Merge branch 'main' into perf/minimax-h3-lossless-hot-path
This commit is contained in:
143 files changed
+8587
-2165
No files matched your search
@@ -90,6 +90,35 @@ class TestAttentionSpec:
|
||||
with pytest.raises(ValueError, match=match):
|
||||
AttentionSpec(**spec)
|
||||
|
||||
def test_block_sparse_defaults_applied_when_backend_selected(self):
|
||||
spec = AttentionSpec(backend="RAINFUSION_ATTN")
|
||||
assert spec.block_sparse.sparsity == 0.8
|
||||
assert spec.backend_kwargs() == {"sparsity": 0.8, "start_step": 0}
|
||||
|
||||
def test_block_sparse_skip_layers_selector_expanded(self):
|
||||
spec = AttentionSpec(
|
||||
backend="RAINFUSION_ATTN",
|
||||
block_sparse={"sparsity": 0.9, "start_step": 12, "skip_layers": "0-2,38"},
|
||||
)
|
||||
assert spec.block_sparse.skip_layer_indices == {0, 1, 2, 38}
|
||||
assert spec.backend_kwargs() == {
|
||||
"sparsity": 0.9,
|
||||
"start_step": 12,
|
||||
"skip_layers": [0, 1, 2, 38],
|
||||
}
|
||||
|
||||
def test_block_sparse_rejected_on_dense_backend(self):
|
||||
with pytest.raises(ValueError, match="block_sparse is only supported by"):
|
||||
AttentionSpec(backend="FLASH_ATTN", block_sparse={"sparsity": 0.8})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block_sparse",
|
||||
[{"sparsity": 1.5}, {"start_step": -1}],
|
||||
)
|
||||
def test_block_sparse_invalid_values(self, block_sparse):
|
||||
with pytest.raises(ValueError):
|
||||
AttentionSpec(backend="RAINFUSION_ATTN", block_sparse=block_sparse)
|
||||
|
||||
|
||||
class TestAttentionConfig:
|
||||
def test_empty_config(self):
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.cudnn_attn import CuDNNAttentionImpl
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
|
||||
|
||||
|
||||
def test_cudnn_slices_valid_kv_prefix_without_padding_mask(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_sdpa(query, key, value, **kwargs):
|
||||
observed.update(query=query, key=key, value=value, kwargs=kwargs)
|
||||
return query
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.attention.backends.cudnn_attn.sdpa_kernel",
|
||||
lambda _backends: nullcontext(),
|
||||
)
|
||||
monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", fake_sdpa)
|
||||
impl = CuDNNAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=4,
|
||||
softmax_scale=0.5,
|
||||
)
|
||||
query = torch.randn(1, 8, 2, 4)
|
||||
key = torch.randn_like(query)
|
||||
value = torch.randn_like(query)
|
||||
|
||||
output = impl.forward_cuda(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
AttentionMetadata(extra={"valid_kv_length": 5}),
|
||||
)
|
||||
|
||||
assert output.shape == query.shape
|
||||
assert observed["query"].shape == (1, 2, 8, 4)
|
||||
assert observed["key"].shape == (1, 2, 5, 4)
|
||||
assert observed["value"].shape == (1, 2, 5, 4)
|
||||
assert observed["kwargs"]["attn_mask"] is None
|
||||
|
||||
|
||||
def test_cudnn_rejects_invalid_valid_kv_length():
|
||||
impl = CuDNNAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=4,
|
||||
softmax_scale=0.5,
|
||||
)
|
||||
query = torch.randn(1, 8, 2, 4)
|
||||
|
||||
with pytest.raises(ValueError, match="valid_kv_length"):
|
||||
impl.forward_cuda(
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
AttentionMetadata(extra={"valid_kv_length": 9}),
|
||||
)
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Geometry contract RAINFUSION_ATTN enforces before handing a forward to rf_v2.
|
||||
|
||||
The grids here are the ones MiniMax-H3 FL2VA actually produces for an 8.7s clip:
|
||||
1280x768 gives a 62x24x40 latent grid whose 59520 video rows land on the 128-row
|
||||
kernel block, and 1344x768 gives 62x24x42 whose 62496 rows straddle it. Only the
|
||||
first can run sparse: rf_v2 tiles the sequence as a whole but pools its block
|
||||
mask from the video and prefix segments separately, so the two only agree on the
|
||||
block count when the video segment ends on a boundary.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
|
||||
from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
|
||||
_BLOCK_SIZE,
|
||||
RainFusionAttentionImpl,
|
||||
RainFusionPlan,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
PREFIX_ROWS = 710 # 14 text rows + 696 audio rows
|
||||
ALIGNED_GRID = (62, 24, 40) # 1280x768 -> 59520 video rows, 465 blocks
|
||||
MISALIGNED_GRID = (62, 24, 42) # 1344x768 -> 62496 video rows, 488 blocks + 32 rows
|
||||
|
||||
|
||||
def make_impl(**backend_kwargs):
|
||||
return RainFusionAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
softmax_scale=128**-0.5,
|
||||
prefix="transformer_blocks.0.attn",
|
||||
qkv_layout="BSND",
|
||||
backend_kwargs={"sparsity": 0.8, **backend_kwargs},
|
||||
)
|
||||
|
||||
|
||||
def make_metadata(grid, prefix_len=PREFIX_ROWS, max_seqlen_q=None):
|
||||
video_rows = grid[0] * grid[1] * grid[2]
|
||||
return AttentionMetadata(
|
||||
extra={"max_seqlen_q": prefix_len + video_rows if max_seqlen_q is None else max_seqlen_q},
|
||||
video_layout=VideoTokenLayout(prefix_len=prefix_len, latent_grid=grid),
|
||||
)
|
||||
|
||||
|
||||
def test_block_aligned_video_segment_runs_sparse():
|
||||
plan = make_impl()._resolve_plan(make_metadata(ALIGNED_GRID))
|
||||
|
||||
assert plan is not None
|
||||
assert plan.prefix_len == PREFIX_ROWS
|
||||
assert plan.used_len == PREFIX_ROWS + 59520
|
||||
assert plan.latent_shape == list(ALIGNED_GRID)
|
||||
|
||||
|
||||
def test_misaligned_video_segment_stays_dense():
|
||||
# Realigning the sequence would mean padding it, and rf_v2 ignores attn_mask:
|
||||
# pad keys would take a share of every real query's softmax denominator.
|
||||
assert make_impl()._resolve_plan(make_metadata(MISALIGNED_GRID)) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix_len", [1, 127, _BLOCK_SIZE, 710, 900])
|
||||
def test_prefix_length_does_not_affect_alignment(prefix_len):
|
||||
# Only the video segment has to land on a block boundary; rf_v2 pools the
|
||||
# prefix separately and keeps every one of its blocks.
|
||||
plan = make_impl()._resolve_plan(make_metadata(ALIGNED_GRID, prefix_len=prefix_len))
|
||||
|
||||
assert plan is not None
|
||||
assert plan.prefix_len == prefix_len
|
||||
|
||||
|
||||
def test_sparsity_zero_never_resolves_a_plan():
|
||||
assert make_impl(sparsity=0.0)._resolve_plan(make_metadata(ALIGNED_GRID)) is None
|
||||
|
||||
|
||||
def test_missing_video_layout_falls_back_to_dense():
|
||||
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 60230})) is None
|
||||
|
||||
|
||||
def test_missing_max_seqlen_falls_back_to_dense():
|
||||
metadata = AttentionMetadata(
|
||||
video_layout=VideoTokenLayout(prefix_len=PREFIX_ROWS, latent_grid=ALIGNED_GRID),
|
||||
)
|
||||
|
||||
assert make_impl()._resolve_plan(metadata) is None
|
||||
|
||||
|
||||
def test_video_segment_must_be_the_tail_of_packed_document_zero():
|
||||
metadata = make_metadata(ALIGNED_GRID, max_seqlen_q=PREFIX_ROWS + 59520 + 128)
|
||||
|
||||
assert make_impl()._resolve_plan(metadata) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
|
||||
def test_short_video_stays_dense(grid):
|
||||
assert make_impl()._resolve_plan(make_metadata(grid)) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qkv_layout", ["BNSD", "BSH"])
|
||||
def test_explicitly_wrong_layout_is_rejected(qkv_layout):
|
||||
with pytest.raises(ValueError, match="BSND"):
|
||||
RainFusionAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
softmax_scale=128**-0.5,
|
||||
qkv_layout=qkv_layout,
|
||||
backend_kwargs={"sparsity": 0.8},
|
||||
)
|
||||
|
||||
|
||||
def test_undeclared_layout_stays_dense():
|
||||
# An undeclared layout is not an error -- the layer keeps working, just
|
||||
# densely, and the fallback sees the same absent layout plain FLASH_ATTN would.
|
||||
impl = RainFusionAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
softmax_scale=128**-0.5,
|
||||
prefix="transformer_blocks.0.attn",
|
||||
backend_kwargs={"sparsity": 0.8},
|
||||
)
|
||||
|
||||
assert impl._resolve_plan(make_metadata(ALIGNED_GRID)) is None
|
||||
assert impl.dense_fallback.qkv_layout is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="rf_v2 runs on Ascend NPU only.")
|
||||
def test_fully_populated_mask_reproduces_dense_attention():
|
||||
"""At sparsity=0 every key block is kept, so rf_v2 must match dense attention.
|
||||
|
||||
This is the guard on the geometry the backend hands the kernel: a wrong
|
||||
prefix length or latent grid still produces plausible output under sparse
|
||||
selection, but shows up here as a mismatch against SDPA.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
prefix_len, grid = 384, (4, 16, 64) # 4096 video rows = 32 blocks
|
||||
video_len = grid[0] * grid[1] * grid[2]
|
||||
used = prefix_len + video_len
|
||||
heads, head_dim = 4, 128
|
||||
q, k, v = (torch.randn(1, used, heads, head_dim, dtype=torch.bfloat16, device="npu") for _ in range(3))
|
||||
|
||||
impl = make_impl()
|
||||
impl.rainfusion = dataclasses.replace(impl.rainfusion, sparsity=0.0)
|
||||
plan = RainFusionPlan(prefix_len=prefix_len, used_len=used, latent_shape=list(grid))
|
||||
out = impl._forward_sparse_npu(q, k, v, plan)
|
||||
|
||||
reference = torch.nn.functional.scaled_dot_product_attention(
|
||||
q.transpose(1, 2),
|
||||
k.transpose(1, 2),
|
||||
v.transpose(1, 2),
|
||||
scale=impl.softmax_scale,
|
||||
).transpose(1, 2)
|
||||
error = (out.float() - reference.float()).abs().mean() / reference.float().abs().mean()
|
||||
assert error < 2e-3, f"mean relative error {error:.4%} against dense attention"
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import trtllm_attn as tg
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]
|
||||
@@ -142,6 +143,9 @@ def test_denoise_progress_mixin(monkeypatch):
|
||||
_Pipe().record_denoise_step(4)
|
||||
assert ctx.denoise_step_idx == 4 and ctx.denoise_timestep == pytest.approx(0.25)
|
||||
|
||||
_Pipe().record_denoise_step(5, normalized_timestep=0.7)
|
||||
assert ctx.denoise_step_idx == 5 and ctx.denoise_timestep == pytest.approx(0.7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bk, field",
|
||||
@@ -187,6 +191,7 @@ def test_mask_free_allowlist_gates_trtllm_default():
|
||||
|
||||
assert meta("WanPipeline").attention_mask_free is True
|
||||
assert meta("WanVACEPipeline").attention_mask_free is True
|
||||
assert meta("MiniMaxH3Pipeline").attention_mask_free is True
|
||||
assert meta("FluxPipeline").attention_mask_free is False
|
||||
assert meta(None).attention_mask_free is False
|
||||
|
||||
@@ -204,6 +209,91 @@ def test_masked_layer_raises():
|
||||
_impl().forward_cuda(q, k, v, _Meta())
|
||||
|
||||
|
||||
def test_packed_metadata_trims_padding_mask(monkeypatch):
|
||||
b, s, h, d = 1, 8, 8, 128
|
||||
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
|
||||
mask = torch.tensor([[True, True, True, True, True, True, False, False]])
|
||||
cu_seqlens = torch.tensor([0, 6, 8], dtype=torch.int32)
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=mask,
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
"max_seqlen_q": 6,
|
||||
"max_seqlen_k": 6,
|
||||
},
|
||||
)
|
||||
captured = {}
|
||||
|
||||
def fake_attention(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return torch.zeros_like(kwargs["query"])
|
||||
|
||||
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
|
||||
monkeypatch.setattr(tg, "trtllm_ragged_attention_deepseek", fake_attention)
|
||||
monkeypatch.setattr(TrtllmAttentionImpl, "_get_workspace", classmethod(lambda cls, device: torch.empty(0)))
|
||||
|
||||
out = _impl().forward_cuda(q, k, v, metadata)
|
||||
|
||||
assert out.shape == q.shape
|
||||
assert captured["query"].shape[0] == 6
|
||||
assert captured["key"].shape[0] == 6
|
||||
assert captured["value"].shape[0] == 6
|
||||
assert captured["batch_size"] == 1
|
||||
assert captured["seq_lens"].tolist() == [6]
|
||||
assert captured["cum_seq_lens_q"].tolist() == [0, 6]
|
||||
assert captured["cum_seq_lens_kv"].tolist() == [0, 6]
|
||||
assert captured["max_q_len"] == 6
|
||||
assert captured["max_kv_len"] == 6
|
||||
assert torch.count_nonzero(out[:, 6:]) == 0
|
||||
|
||||
|
||||
def test_packed_metadata_rejects_non_prefix_mask(monkeypatch):
|
||||
b, s, h, d = 1, 8, 8, 128
|
||||
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
|
||||
cu_seqlens = torch.tensor([0, 6, 8], dtype=torch.int32)
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=torch.tensor([[True, True, False, True, True, True, False, False]]),
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
"max_seqlen_q": 6,
|
||||
"max_seqlen_k": 6,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
|
||||
|
||||
with pytest.raises(ValueError, match="prefix-valid"):
|
||||
_impl().forward_cuda(q, k, v, metadata)
|
||||
|
||||
|
||||
def test_packed_metadata_must_be_complete():
|
||||
b, s, h, d = 1, 8, 8, 128
|
||||
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
|
||||
metadata = AttentionMetadata(extra={"cu_seqlens_q": torch.tensor([0, s], dtype=torch.int32)})
|
||||
|
||||
with pytest.raises(ValueError, match="Incomplete packed TRTLLM attention metadata"):
|
||||
_impl().forward_cuda(q, k, v, metadata)
|
||||
|
||||
|
||||
def test_packed_metadata_must_cover_inputs(monkeypatch):
|
||||
b, s, h, d = 1, 8, 8, 128
|
||||
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
|
||||
cu_seqlens = torch.tensor([0, 6], dtype=torch.int32)
|
||||
metadata = AttentionMetadata(
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
"max_seqlen_q": 6,
|
||||
"max_seqlen_k": 6,
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
|
||||
|
||||
with pytest.raises(ValueError, match="must cover all Q/K/V tokens"):
|
||||
_impl().forward_cuda(q, k, v, metadata)
|
||||
|
||||
|
||||
def test_propagate_calibration_045_schema(monkeypatch):
|
||||
import types
|
||||
|
||||
@@ -295,6 +385,30 @@ def test_bf16_dense_matches_sdpa():
|
||||
assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"
|
||||
|
||||
|
||||
@requires_trtllm_attn
|
||||
def test_bf16_packed_padding_matches_sdpa():
|
||||
torch.manual_seed(0)
|
||||
b, s, used, h, d = 1, 256, 192, 8, 128
|
||||
scale = 1.0 / math.sqrt(d)
|
||||
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
|
||||
cu_seqlens = torch.tensor([0, used, s], dtype=torch.int32, device="cuda")
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=torch.arange(s, device="cuda")[None] < used,
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
"max_seqlen_q": used,
|
||||
"max_seqlen_k": used,
|
||||
},
|
||||
)
|
||||
|
||||
out = _impl().forward_cuda(q, k, v, metadata).float()
|
||||
ref_used = _sdpa_ref(q[:, :used], k[:, :used], v[:, :used], scale)
|
||||
used_rel = (out[:, :used] - ref_used).abs().mean() / ref_used.abs().mean()
|
||||
assert used_rel < 0.01, f"BF16 packed valid-token rel err {used_rel:.4f} too high"
|
||||
assert torch.count_nonzero(out[:, used:]) == 0
|
||||
|
||||
|
||||
requires_sage = pytest.mark.skipif(
|
||||
not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),
|
||||
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)",
|
||||
@@ -321,6 +435,47 @@ def test_sage_quant_matches_sdpa(dtype_qk):
|
||||
assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"
|
||||
|
||||
|
||||
@requires_sage
|
||||
def test_sage_packed_non_aligned_length_matches_sdpa():
|
||||
torch.manual_seed(0)
|
||||
b, s, used, h, d = 1, 208, 194, 8, 128
|
||||
scale = 1.0 / math.sqrt(d)
|
||||
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
|
||||
cu_seqlens = torch.tensor([0, used, s], dtype=torch.int32, device="cuda")
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=torch.arange(s, device="cuda")[None] < used,
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
"max_seqlen_q": used,
|
||||
"max_seqlen_k": used,
|
||||
},
|
||||
)
|
||||
|
||||
out = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}).forward_cuda(q, k, v, metadata)
|
||||
assert torch.isfinite(out).all()
|
||||
ref = _sdpa_ref(q[:, :used], k[:, :used], v[:, :used], scale)
|
||||
rel = (out[:, :used].float() - ref).abs().mean() / ref.abs().mean()
|
||||
assert rel < 0.2, f"SAGE packed valid-token rel err {rel:.4f} too high"
|
||||
assert torch.count_nonzero(out[:, used:]) == 0
|
||||
|
||||
|
||||
@requires_sage
|
||||
def test_sage_short_sequence_uses_dense_kernel():
|
||||
torch.manual_seed(0)
|
||||
b, s, h, d = 1, 14, 8, 128
|
||||
scale = 1.0 / math.sqrt(d)
|
||||
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
|
||||
|
||||
impl = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16})
|
||||
impl._sage_quantize_fn = lambda *args, **kwargs: pytest.fail("short sequence must not use SAGE")
|
||||
out = impl.forward_cuda(q, k, v)
|
||||
ref = _sdpa_ref(q, k, v, scale)
|
||||
rel = (out.float() - ref).abs().mean() / ref.abs().mean()
|
||||
assert torch.isfinite(out).all()
|
||||
assert rel < 0.01, f"SAGE short-sequence dense fallback rel err {rel:.4f} too high"
|
||||
|
||||
|
||||
@requires_trtllm_attn
|
||||
def test_skip_tiny_threshold_matches_dense():
|
||||
torch.manual_seed(0)
|
||||
|
||||
+31
-5
@@ -86,9 +86,35 @@ class TestCacheDiTBackend:
|
||||
|
||||
# Verify cache-dit was enabled
|
||||
assert backend.enabled is True
|
||||
assert backend._refresh_func is not None
|
||||
assert backend._refresh_funcs
|
||||
mock_cache_dit.enable_cache.assert_called_once()
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_and_refresh_every_declared_dit(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Combined pipelines enable and refresh Cache-DiT for every DiT."""
|
||||
pipeline = Mock()
|
||||
pipeline.__class__.__name__ = "MiniMaxH3Pipeline"
|
||||
pipeline._dit_modules = ["transformer", "transformers_ref"]
|
||||
pipeline.transformer = Mock()
|
||||
pipeline.transformers_ref = Mock()
|
||||
for transformer in (pipeline.transformer, pipeline.transformers_ref):
|
||||
transformer._cache_dit_adapter_config = CacheDiTAdapterConfig(
|
||||
block_forward_patterns={"blocks": ForwardPattern.Pattern_3}
|
||||
)
|
||||
|
||||
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
|
||||
backend.enable(pipeline)
|
||||
backend.refresh(pipeline, num_inference_steps=20)
|
||||
|
||||
assert len(backend._refresh_funcs) == 2
|
||||
assert mock_cache_dit.enable_cache.call_count == 2
|
||||
assert mock_cache_dit.refresh_context.call_count == 2
|
||||
assert {call.args[0] for call in mock_cache_dit.refresh_context.call_args_list} == {
|
||||
pipeline.transformer,
|
||||
pipeline.transformers_ref,
|
||||
}
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.logger")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_without_model_adapter_uses_cache_dit_registry(self, mock_cache_dit, mock_logger):
|
||||
@@ -106,7 +132,7 @@ class TestCacheDiTBackend:
|
||||
backend.enable(pipeline)
|
||||
|
||||
assert backend.enabled is True
|
||||
assert backend._refresh_func is not None
|
||||
assert backend._refresh_funcs
|
||||
assert mock_cache_dit.enable_cache.call_args.args[0] is pipeline.transformer
|
||||
mock_logger.info.assert_any_call(
|
||||
"Transformer %s does not declare _cache_dit_adapter_config; "
|
||||
@@ -138,7 +164,7 @@ class TestCacheDiTBackend:
|
||||
|
||||
assert isinstance(exc_info.value.__cause__, ValueError)
|
||||
assert backend.enabled is False
|
||||
assert backend._refresh_func is None
|
||||
assert backend._refresh_funcs == []
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
@@ -245,7 +271,7 @@ class TestCacheDiTBackend:
|
||||
backend.enable(mock_pipeline)
|
||||
|
||||
assert backend.enabled is True
|
||||
assert backend._refresh_func is not None
|
||||
assert backend._refresh_funcs
|
||||
mock_block_adapter.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
assert adapter_kwargs["transformer"] is mock_pipeline.model
|
||||
@@ -285,7 +311,7 @@ class TestCacheDiTBackend:
|
||||
backend.enable(mock_pipeline)
|
||||
|
||||
assert backend.enabled is True
|
||||
assert backend._refresh_func is not None
|
||||
assert backend._refresh_funcs
|
||||
mock_block_adapter.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
assert adapter_kwargs["transformer"] is mock_pipeline.transformer
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for LayerNorm and RMSNorm custom ops in diffusion layers."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -299,6 +302,44 @@ def test_rmsnorm_numerical_correctness():
|
||||
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
def test_rmsnorm_dtype_matches_parameter_and_fp32_accumulation_contract():
|
||||
"""dtype controls gamma storage; reduced-precision inputs still use fp32 accumulation."""
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
|
||||
hidden_size = 64
|
||||
eps = 1e-6
|
||||
norm = RMSNorm(hidden_size, eps=eps, dtype=torch.bfloat16)
|
||||
x = torch.randn(2, 4, hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
assert norm.weight.dtype == torch.bfloat16
|
||||
|
||||
x_fp32 = x.float()
|
||||
expected = (x_fp32 * torch.rsqrt(x_fp32.square().mean(-1, keepdim=True) + eps) * norm.weight.float()).to(x.dtype)
|
||||
|
||||
torch.testing.assert_close(norm.forward_native(x), expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_rmsnorm_npu_receives_gamma_with_configured_dtype(monkeypatch: pytest.MonkeyPatch):
|
||||
"""The NPU fused op receives the bf16 checkpoint gamma used by H3."""
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
|
||||
captured: dict[str, torch.Tensor] = {}
|
||||
|
||||
def npu_rms_norm(x: torch.Tensor, gamma: torch.Tensor, epsilon: float):
|
||||
captured["gamma"] = gamma
|
||||
captured["epsilon"] = torch.tensor(epsilon)
|
||||
return (x,)
|
||||
|
||||
monkeypatch.setitem(sys.modules, "torch_npu", types.SimpleNamespace(npu_rms_norm=npu_rms_norm))
|
||||
norm = RMSNorm(64, eps=1e-5, dtype=torch.bfloat16)
|
||||
x = torch.randn(2, 4, 64, dtype=torch.bfloat16)
|
||||
|
||||
assert norm.forward_npu(x) is x
|
||||
assert captured["gamma"] is norm.weight
|
||||
assert captured["gamma"].dtype == torch.bfloat16
|
||||
assert captured["epsilon"].item() == pytest.approx(1e-5)
|
||||
|
||||
|
||||
# ── RMSNorm compile-path regression tests ──
|
||||
|
||||
|
||||
|
||||
@@ -10,10 +10,13 @@ forward_hip, and apply_rotary_emb_mindiesd paths.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rotary_emb_mindiesd
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -234,3 +237,33 @@ class TestRotaryEmbeddingNativeShapeRegression:
|
||||
output = rope.forward_native(x, cos, sin)
|
||||
assert output.shape == x.shape
|
||||
assert not torch.isnan(output).any()
|
||||
|
||||
|
||||
def test_mindie_full_dim_rope_keeps_h3_cos_sin_layout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""H3 passes complete rotary dims to MindIE instead of the CUDA half-dim layout."""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def rotary_position_embedding(x, cos, sin, **kwargs):
|
||||
captured.update(x=x, cos=cos, sin=sin, kwargs=kwargs)
|
||||
return x
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"mindiesd",
|
||||
types.SimpleNamespace(rotary_position_embedding=rotary_position_embedding),
|
||||
)
|
||||
x = torch.randn(11, 3, 96, dtype=torch.bfloat16)
|
||||
cos = torch.randn(11, 96, dtype=torch.bfloat16)
|
||||
sin = torch.randn(11, 96, dtype=torch.bfloat16)
|
||||
|
||||
actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)
|
||||
|
||||
assert actual is x
|
||||
assert captured["x"] is x
|
||||
assert captured["cos"] is cos
|
||||
assert captured["sin"] is sin
|
||||
assert captured["kwargs"] == {
|
||||
"rotated_mode": "rotated_half",
|
||||
"head_first": False,
|
||||
"fused": True,
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
@@ -24,12 +25,343 @@ def test_pipeline_import_registry_and_component_discovery():
|
||||
"pipeline_minimax_h3",
|
||||
"MiniMaxH3Pipeline",
|
||||
)
|
||||
assert _DIFFUSION_MODELS["MiniMaxH3ModularPipeline"] == _DIFFUSION_MODELS["MiniMaxH3Pipeline"]
|
||||
assert _DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3Pipeline"] == "get_minimax_h3_post_process_func"
|
||||
assert MiniMaxH3Pipeline._dit_modules == ["transformer"]
|
||||
assert (
|
||||
_DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3ModularPipeline"] == _DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3Pipeline"]
|
||||
)
|
||||
assert MiniMaxH3Pipeline._dit_modules == ["transformer", "transformers_ref"]
|
||||
assert MiniMaxH3Pipeline._encoder_modules == ["text_encoder"]
|
||||
assert MiniMaxH3Pipeline._vae_modules == ["video_vae", "audio_vae"]
|
||||
|
||||
|
||||
def _write_partition_index(path, *, partition, tasks):
|
||||
path.mkdir(parents=True)
|
||||
(path / "model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": "MiniMaxH3Pipeline",
|
||||
"_diffusers_version": "0.32.2",
|
||||
"_minimax_h3": {
|
||||
"partition": partition,
|
||||
"tasks": tasks,
|
||||
"sigma_shift_scales": {"video": 12.0, "audio": 3.0},
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_modular_diffusers_index_is_resolved_generically(tmp_path):
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, resolve_model_class_name
|
||||
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
|
||||
from vllm_omni.entrypoints.utils import resolve_model_config_path
|
||||
|
||||
(tmp_path / "modular_model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": "MiniMaxH3ModularPipeline",
|
||||
"_diffusers_version": "0.36.0.dev0",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert is_diffusion_model(str(tmp_path))
|
||||
assert resolve_model_class_name(str(tmp_path)) == "MiniMaxH3ModularPipeline"
|
||||
assert resolve_model_config_path(str(tmp_path)) is None
|
||||
|
||||
config = OmniDiffusionConfig(model=str(tmp_path))
|
||||
config.enrich_config()
|
||||
assert config.model_class_name == "MiniMaxH3ModularPipeline"
|
||||
assert config.supports_multimodal_inputs
|
||||
assert config.max_multimodal_image_inputs == 9
|
||||
assert config.supports_mixed_reference_inputs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("task_type", "partition"),
|
||||
[
|
||||
(None, "combined"),
|
||||
("auto", "combined"),
|
||||
("t2va", "fl2va"),
|
||||
("fl2va", "fl2va"),
|
||||
("ref2va", "ref2va"),
|
||||
],
|
||||
)
|
||||
def test_startup_task_selects_weight_partition(task_type, partition):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
|
||||
|
||||
assert _minimax_h3_partition_for_task(task_type) == partition
|
||||
|
||||
|
||||
def test_startup_auto_task_uses_explicit_local_partition(tmp_path):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
|
||||
|
||||
for directory, partition, tasks in (
|
||||
("FL2VA", "fl2va", ["t2va", "fl2va"]),
|
||||
("Ref2VA", "ref2va", ["ref2va"]),
|
||||
):
|
||||
model_path = tmp_path / directory
|
||||
_write_partition_index(model_path, partition=partition, tasks=tasks)
|
||||
assert _minimax_h3_partition_for_task(None, str(model_path)) == partition
|
||||
assert _minimax_h3_partition_for_task("auto", str(model_path)) == partition
|
||||
assert _minimax_h3_partition_for_task("combined", str(model_path)) == "combined"
|
||||
|
||||
|
||||
def test_startup_task_rejects_unsupported_value():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
|
||||
|
||||
with pytest.raises(ValueError, match="task_type must be one of"):
|
||||
_minimax_h3_partition_for_task("unsupported")
|
||||
|
||||
|
||||
def test_combined_task_inference_and_transformer_routing():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.partition = "combined"
|
||||
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
|
||||
pipeline.transformer = torch.nn.Identity()
|
||||
pipeline.transformers_ref = torch.nn.Linear(1, 1)
|
||||
assert pipeline._resolve_task(None, {}) == "t2va"
|
||||
assert pipeline._resolve_task(None, {"image": object()}) == "fl2va"
|
||||
assert pipeline._resolve_task(None, {"audio": object()}) == "ref2va"
|
||||
assert pipeline._resolve_task(None, {"video": object()}) == "ref2va"
|
||||
pipeline.partition = "ref2va"
|
||||
pipeline.supported_tasks = frozenset({"ref2va"})
|
||||
assert pipeline._resolve_task(None, {"image": object()}) == "ref2va"
|
||||
|
||||
pipeline.partition = "combined"
|
||||
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
|
||||
assert pipeline._transformer_for_task("fl2va") is pipeline.transformer
|
||||
assert pipeline._transformer_for_task("ref2va") is pipeline.transformers_ref
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"task_type",
|
||||
"expected_partition",
|
||||
"expected_dits",
|
||||
"component_partition",
|
||||
"source_partitions",
|
||||
"expected_tasks",
|
||||
),
|
||||
[
|
||||
(None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}),
|
||||
("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}),
|
||||
("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}),
|
||||
],
|
||||
)
|
||||
def test_pipeline_loads_task_selected_dits_and_shared_components_once(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
task_type,
|
||||
expected_partition,
|
||||
expected_dits,
|
||||
component_partition,
|
||||
source_partitions,
|
||||
expected_tasks,
|
||||
):
|
||||
from vllm_omni.diffusion.data import (
|
||||
DiffusionParallelConfig,
|
||||
OmniDiffusionConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3 import (
|
||||
pipeline_minimax_h3 as pipeline_module,
|
||||
)
|
||||
|
||||
_write_partition_index(
|
||||
tmp_path / "FL2VA",
|
||||
partition="fl2va",
|
||||
tasks=["t2va", "fl2va"],
|
||||
)
|
||||
_write_partition_index(
|
||||
tmp_path / "Ref2VA",
|
||||
partition="ref2va",
|
||||
tasks=["ref2va"],
|
||||
)
|
||||
for partition_name in ("FL2VA", "Ref2VA"):
|
||||
for component in (
|
||||
"transformer",
|
||||
"tokenizer",
|
||||
"processor",
|
||||
"text_encoder",
|
||||
"video_vae",
|
||||
"audio_vae",
|
||||
):
|
||||
(tmp_path / partition_name / component).mkdir()
|
||||
|
||||
created = {"dit": [], "text_encoder": [], "video_vae": [], "audio_vae": []}
|
||||
|
||||
class FakeModule(torch.nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
class FakeDiT(FakeModule):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
created["dit"].append((args, kwargs))
|
||||
|
||||
def component_factory(name):
|
||||
def create(path, *args, **kwargs):
|
||||
created[name].append(str(path))
|
||||
return FakeModule()
|
||||
|
||||
return create
|
||||
|
||||
tokenizer_calls = []
|
||||
processor_calls = []
|
||||
tokenizer_cls = Mock(spec=pipeline_module.Qwen2TokenizerFast)
|
||||
tokenizer_cls.from_pretrained.side_effect = lambda *args, **kwargs: (
|
||||
tokenizer_calls.append((args, kwargs)) or object()
|
||||
)
|
||||
processor_cls = Mock(spec=pipeline_module.Qwen3VLProcessor)
|
||||
processor_cls.from_pretrained.side_effect = lambda *args, **kwargs: (
|
||||
processor_calls.append((args, kwargs)) or object()
|
||||
)
|
||||
monkeypatch.setattr(pipeline_module, "MiniMaxH3DiTModel", FakeDiT)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"MiniMaxH3Qwen3VLEncoder",
|
||||
component_factory("text_encoder"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"MiniMaxH3VideoVAE",
|
||||
component_factory("video_vae"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"MiniMaxH3AudioVAE",
|
||||
component_factory("audio_vae"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"Qwen2TokenizerFast",
|
||||
tokenizer_cls,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"Qwen3VLProcessor",
|
||||
processor_cls,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"_dit_rank_world",
|
||||
lambda: (None, 0, 1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"get_local_device",
|
||||
lambda: torch.device("cpu"),
|
||||
)
|
||||
download_calls = []
|
||||
|
||||
def fake_download(**kwargs):
|
||||
download_calls.append(kwargs)
|
||||
return str(tmp_path)
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_module,
|
||||
"download_weights_from_hf_specific",
|
||||
fake_download,
|
||||
)
|
||||
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="MiniMaxAI/MiniMax-H3",
|
||||
revision=None,
|
||||
task_type=task_type,
|
||||
quantization_config=None,
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
cfg_parallel_size=1,
|
||||
text_encoder_tp_size=1,
|
||||
),
|
||||
)
|
||||
pipeline = pipeline_module.MiniMaxH3Pipeline(od_config=od_config)
|
||||
|
||||
assert pipeline.partition == expected_partition
|
||||
assert pipeline.supported_tasks == expected_tasks
|
||||
assert len(created["dit"]) == expected_dits
|
||||
component_path = tmp_path / component_partition
|
||||
assert created["text_encoder"] == [str(component_path / "text_encoder")]
|
||||
assert created["video_vae"] == [str(component_path / "video_vae")]
|
||||
assert created["audio_vae"] == [str(component_path / "audio_vae")]
|
||||
assert len(tokenizer_calls) == 1
|
||||
assert len(processor_calls) == 1
|
||||
expected_dit_modules = ["transformer", "transformers_ref"] if expected_dits == 2 else ["transformer"]
|
||||
assert pipeline._dit_modules == expected_dit_modules
|
||||
assert hasattr(pipeline, "transformers_ref") is (expected_dits == 2)
|
||||
if expected_partition == "ref2va":
|
||||
assert pipeline._transformer_for_task("ref2va") is pipeline.transformer
|
||||
assert [source.model_or_path for source in pipeline.weights_sources] == [
|
||||
str(tmp_path / partition_name) for partition_name in source_partitions
|
||||
]
|
||||
expected_patterns = (
|
||||
pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS
|
||||
if expected_partition == "combined"
|
||||
else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition]
|
||||
)
|
||||
assert download_calls == [
|
||||
{
|
||||
"model_name_or_path": "MiniMaxAI/MiniMax-H3",
|
||||
"cache_dir": None,
|
||||
"allow_patterns": expected_patterns,
|
||||
"revision": None,
|
||||
"require_all": True,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_combined_weight_loader_routes_each_contiguous_partition():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
class FakeTransformer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.loaded = []
|
||||
self.post_load_calls = 0
|
||||
|
||||
def load_weights(self, weights):
|
||||
self.loaded = [name for name, _ in weights]
|
||||
return set(self.loaded)
|
||||
|
||||
def post_load_weights(self):
|
||||
self.post_load_calls += 1
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.transformer = FakeTransformer()
|
||||
pipeline.transformers_ref = FakeTransformer()
|
||||
pipeline.text_encoder = torch.nn.Identity()
|
||||
pipeline.video_vae = torch.nn.Identity()
|
||||
pipeline.audio_vae = torch.nn.Identity()
|
||||
loaded = pipeline.load_weights(
|
||||
iter(
|
||||
[
|
||||
("transformer.a", torch.ones(1)),
|
||||
("transformer.b", torch.ones(1)),
|
||||
("transformers_ref.a", torch.ones(1)),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
assert pipeline.transformer.loaded == ["a", "b"]
|
||||
assert pipeline.transformers_ref.loaded == ["a"]
|
||||
assert pipeline.transformer.post_load_calls == 1
|
||||
assert pipeline.transformers_ref.post_load_calls == 1
|
||||
assert loaded == {"transformer.a", "transformer.b", "transformers_ref.a"}
|
||||
|
||||
|
||||
def test_dlo_offload_plan_includes_token_refiner():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
assert MiniMaxH3Pipeline._offload_plan.offload_submodules == {"token_refiner": "blocks"}
|
||||
assert MiniMaxH3Pipeline._offload_plan.resident_dit_paths == frozenset({"transformer"})
|
||||
|
||||
|
||||
def test_joint_postprocess_is_multiprocessing_picklable():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import (
|
||||
get_minimax_h3_post_process_func,
|
||||
@@ -87,6 +419,87 @@ def test_shifted_sigma_schedule_matches_reference_values():
|
||||
)
|
||||
|
||||
|
||||
def test_cudnn_packed_attention_uses_python_length_without_padding_mask():
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
MiniMaxH3Attention,
|
||||
)
|
||||
|
||||
class FakeBackend:
|
||||
supports_prefix_kv_slicing = True
|
||||
|
||||
class FakeAttention(torch.nn.Module):
|
||||
attn_backend = FakeBackend
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.metadata = None
|
||||
|
||||
def forward(self, query, key, value, metadata):
|
||||
self.metadata = metadata
|
||||
return query
|
||||
|
||||
attention = object.__new__(MiniMaxH3Attention)
|
||||
torch.nn.Module.__init__(attention)
|
||||
attention.attention = FakeAttention()
|
||||
# Model-side Ulysses shards rows before Attention gathers them again.
|
||||
# The Python packed_total must therefore remain global even though q is
|
||||
# local here.
|
||||
q = torch.randn(4, 2, 4)
|
||||
cu_seqlens = torch.tensor([0, 5, 8], dtype=torch.int32)
|
||||
|
||||
output = attention._run_packed_attention(
|
||||
q,
|
||||
q,
|
||||
q,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=5,
|
||||
packed_total=8,
|
||||
)
|
||||
|
||||
assert output.shape == q.shape
|
||||
assert attention.attention.metadata.attn_mask is None
|
||||
assert attention.attention.metadata.extra["valid_kv_length"] == 5
|
||||
|
||||
|
||||
def test_packed_attention_keeps_padding_mask_for_other_backends():
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
MiniMaxH3Attention,
|
||||
)
|
||||
|
||||
class FakeBackend:
|
||||
supports_prefix_kv_slicing = False
|
||||
|
||||
class FakeAttention(torch.nn.Module):
|
||||
attn_backend = FakeBackend
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.metadata = None
|
||||
|
||||
def forward(self, query, key, value, metadata):
|
||||
self.metadata = metadata
|
||||
return query
|
||||
|
||||
attention = object.__new__(MiniMaxH3Attention)
|
||||
torch.nn.Module.__init__(attention)
|
||||
attention.attention = FakeAttention()
|
||||
q = torch.randn(8, 2, 4)
|
||||
|
||||
attention._run_packed_attention(
|
||||
q,
|
||||
q,
|
||||
q,
|
||||
cu_seqlens=torch.tensor([0, 5, 8], dtype=torch.int32),
|
||||
max_seqlen=5,
|
||||
packed_total=8,
|
||||
)
|
||||
|
||||
assert torch.equal(
|
||||
attention.attention.metadata.attn_mask,
|
||||
torch.tensor([[True, True, True, True, True, False, False, False]]),
|
||||
)
|
||||
|
||||
|
||||
def test_reference_image_resize_contract():
|
||||
from PIL import Image
|
||||
|
||||
@@ -293,6 +706,142 @@ def test_layerwise_offload_releases_text_encoder():
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_offload_releases_text_encoder():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=True,
|
||||
)
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_offload_stages_vae_component():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=True,
|
||||
)
|
||||
component = Mock()
|
||||
|
||||
with pipeline._component_on_device(component):
|
||||
component.load_to_device.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
|
||||
component.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_resident_blocks_are_stage_scoped():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
controller = Mock()
|
||||
pipeline._dlo_residency_controller = controller
|
||||
|
||||
with pipeline._resident_dit_layers_on_device():
|
||||
controller.load_resident_layers.assert_called_once_with()
|
||||
controller.offload_resident_layers.assert_not_called()
|
||||
|
||||
controller.offload_resident_layers.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_resident_blocks_release_on_failure():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
controller = Mock()
|
||||
pipeline._dlo_residency_controller = controller
|
||||
|
||||
with pytest.raises(RuntimeError, match="denoise failed"):
|
||||
with pipeline._resident_dit_layers_on_device():
|
||||
raise RuntimeError("denoise failed")
|
||||
|
||||
controller.load_resident_layers.assert_called_once_with()
|
||||
controller.offload_resident_layers.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_resident_blocks_can_be_skipped():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
controller = Mock()
|
||||
pipeline._dlo_residency_controller = controller
|
||||
|
||||
with pipeline._resident_dit_layers_on_device(enabled=False):
|
||||
pass
|
||||
|
||||
controller.load_resident_layers.assert_not_called()
|
||||
controller.offload_resident_layers.assert_not_called()
|
||||
|
||||
|
||||
def test_encoder_layerwise_offload_keeps_tp_blocks_rank_local(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3.encoder import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
)
|
||||
|
||||
class Stack(torch.nn.Module):
|
||||
def __init__(self, count):
|
||||
super().__init__()
|
||||
self.blocks = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
|
||||
|
||||
class TextStack(torch.nn.Module):
|
||||
def __init__(self, count):
|
||||
super().__init__()
|
||||
self.layers = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
|
||||
|
||||
hooks = []
|
||||
|
||||
def fake_apply(block, next_block, device, stream, pin_memory):
|
||||
hook = SimpleNamespace(
|
||||
block=block,
|
||||
next_block=next_block,
|
||||
device=device,
|
||||
stream=stream,
|
||||
pin_memory=pin_memory,
|
||||
_prev_hook=None,
|
||||
offload_layer=Mock(),
|
||||
)
|
||||
hooks.append(hook)
|
||||
return hook
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.offloader.layerwise_backend.apply_block_hook",
|
||||
fake_apply,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.platforms.current_omni_platform.Stream",
|
||||
Mock(return_value="copy-stream"),
|
||||
)
|
||||
encoder = object.__new__(MiniMaxH3Qwen3VLEncoder)
|
||||
torch.nn.Module.__init__(encoder)
|
||||
encoder.device_target = torch.device("cpu")
|
||||
encoder.vision = Stack(2)
|
||||
encoder.text_model = TextStack(3)
|
||||
|
||||
encoder.enable_omni_layerwise_offload(pin_memory=False)
|
||||
|
||||
assert len(hooks) == 5
|
||||
assert all(hook._prev_hook is not None for hook in hooks)
|
||||
assert all(hook.device == torch.device("cpu") for hook in hooks)
|
||||
|
||||
|
||||
def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
|
||||
|
||||
@@ -324,6 +873,82 @@ def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
|
||||
assert next(video_vae.parameters()).dtype == torch.float32
|
||||
|
||||
|
||||
def test_video_vae_can_load_on_cpu_for_staged_gpu_residency(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
|
||||
|
||||
class FakeRemote(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = torch.nn.Linear(1, 1)
|
||||
|
||||
monkeypatch.setattr(
|
||||
vae_module,
|
||||
"_load_component_config",
|
||||
lambda _path: {
|
||||
"latent_channels": 1,
|
||||
"latents_mean": [0.0],
|
||||
"latents_std": [1.0],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vae_module,
|
||||
"_load_remote_component",
|
||||
lambda _path, _config: FakeRemote(),
|
||||
)
|
||||
|
||||
video_vae = vae_module.MiniMaxH3VideoVAE(
|
||||
"unused",
|
||||
device=torch.device("meta"),
|
||||
load_device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert next(video_vae.parameters()).device.type == "cpu"
|
||||
assert video_vae._device_target.type == "meta"
|
||||
|
||||
|
||||
def test_video_vae_uses_snapshot_stager_for_accelerator_target(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
|
||||
|
||||
class FakeRemote(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = torch.nn.Linear(1, 1)
|
||||
|
||||
stager = Mock()
|
||||
stager_cls = Mock(return_value=stager)
|
||||
monkeypatch.setattr(
|
||||
vae_module,
|
||||
"_load_component_config",
|
||||
lambda _path: {
|
||||
"latent_channels": 1,
|
||||
"latents_mean": [0.0],
|
||||
"latents_std": [1.0],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vae_module,
|
||||
"_load_remote_component",
|
||||
lambda _path, _config: FakeRemote(),
|
||||
)
|
||||
monkeypatch.setattr(vae_module, "PinnedModuleStager", stager_cls)
|
||||
|
||||
video_vae = vae_module.MiniMaxH3VideoVAE(
|
||||
"unused",
|
||||
device=torch.device("cuda"),
|
||||
load_device=torch.device("cpu"),
|
||||
)
|
||||
video_vae.load_to_device()
|
||||
video_vae.offload_to_cpu()
|
||||
|
||||
stager_cls.assert_called_once_with(
|
||||
video_vae.remote,
|
||||
torch.device("cuda"),
|
||||
pin_memory=True,
|
||||
)
|
||||
stager.load.assert_called_once_with()
|
||||
stager.offload.assert_called_once_with()
|
||||
|
||||
|
||||
def test_video_vae_encode_uses_configured_parallel_tiling():
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||
MiniMaxH3VideoVAE,
|
||||
|
||||
@@ -61,6 +61,33 @@ def test_packed_attention_is_a_regional_compile_boundary():
|
||||
assert getattr(MiniMaxH3Attention._run_packed_attention, "_torchdynamo_disable", False)
|
||||
|
||||
|
||||
def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims():
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
MiniMaxH3Attention,
|
||||
)
|
||||
|
||||
attention = object.__new__(MiniMaxH3Attention)
|
||||
nn.Module.__init__(attention)
|
||||
attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
|
||||
attention.rope._forward_method = attention.rope.forward_native
|
||||
|
||||
x = torch.randn(11, 3, 128, dtype=torch.bfloat16)
|
||||
freqs_half = torch.randn(11, 48)
|
||||
freqs = torch.cat((freqs_half, freqs_half), dim=-1)
|
||||
actual = attention._apply_rope(x, freqs)
|
||||
|
||||
cos = torch.cos(freqs).to(x.dtype).unsqueeze(1)
|
||||
sin = torch.sin(freqs).to(x.dtype).unsqueeze(1)
|
||||
x_rot = x[..., :96]
|
||||
x1, x2 = x_rot.chunk(2, dim=-1)
|
||||
expected_rot = x_rot * cos + torch.cat((-x2, x1), dim=-1) * sin
|
||||
expected = torch.cat((expected_rot, x[..., 96:]), dim=-1)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual[..., 96:], x[..., 96:], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tp_size", "message"),
|
||||
[
|
||||
|
||||
@@ -135,6 +135,7 @@ def _generate_joint_output(omni, config: QualityTestConfig):
|
||||
num_frames=config.num_frames,
|
||||
guidance_scale=config.guidance_scale,
|
||||
sigmas=config.sigmas,
|
||||
extra_args={"aspect_ratio": "16:9"},
|
||||
),
|
||||
)
|
||||
main_process_peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
|
||||
|
||||
@@ -27,7 +27,9 @@ from vllm_omni.diffusion.offloader.block_discovery import (
|
||||
from vllm_omni.diffusion.offloader.distributed_layerwise_backend import (
|
||||
DistributedLayerwiseOffloadBackend,
|
||||
DistributedLayerwiseOffloadHook,
|
||||
PinnedResidentLayerGroup,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager
|
||||
from vllm_omni.diffusion.offloader.offload_plan import OffloadPlan, get_offload_plan
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -97,6 +99,7 @@ def patched_offload_runtime(monkeypatch):
|
||||
monkeypatch.setattr(dist_backend_module.current_omni_platform, "Event", DummyEvent)
|
||||
monkeypatch.setattr(dist_backend_module.current_omni_platform, "current_stream", lambda: DummyStream())
|
||||
monkeypatch.setattr(dist_backend_module.current_omni_platform, "stream", dummy_stream)
|
||||
monkeypatch.setattr(dist_backend_module.current_omni_platform, "synchronize", lambda: None)
|
||||
|
||||
|
||||
class TinyBlock(nn.Module):
|
||||
@@ -308,6 +311,164 @@ class TestDistributedLayerwiseOffloadHook:
|
||||
assert shard1.numel() == 2
|
||||
assert torch.equal(shard1, torch.tensor([102.0, 0.0]))
|
||||
|
||||
def test_mmap_layout_transform_is_applied_before_sharding(self, dist_group, patched_offload_runtime):
|
||||
block = nn.Module()
|
||||
block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
|
||||
next_block = nn.Module()
|
||||
next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32))
|
||||
next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0)
|
||||
next_block.weight.mmap_weight_transform_pending = True
|
||||
|
||||
hook = DistributedLayerwiseOffloadHook(
|
||||
next_block=next_block,
|
||||
device=torch.device("cpu"),
|
||||
dp_group=None,
|
||||
dp_size=2,
|
||||
rank=0,
|
||||
pin_memory=False,
|
||||
)
|
||||
hook.initialize_hook(block)
|
||||
|
||||
assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([3.0, 2.0]))
|
||||
|
||||
def test_regular_loader_weight_does_not_reapply_mmap_transform(self, dist_group, patched_offload_runtime):
|
||||
block = nn.Module()
|
||||
block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
|
||||
next_block = nn.Module()
|
||||
next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32))
|
||||
next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0)
|
||||
|
||||
hook = DistributedLayerwiseOffloadHook(
|
||||
next_block=next_block,
|
||||
device=torch.device("cpu"),
|
||||
dp_group=None,
|
||||
dp_size=2,
|
||||
rank=0,
|
||||
pin_memory=False,
|
||||
)
|
||||
hook.initialize_hook(block)
|
||||
|
||||
assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([0.0, 1.0]))
|
||||
|
||||
|
||||
class TestPinnedResidentLayerGroup:
|
||||
def test_load_offload_reuses_pinned_master_weights(self, patched_offload_runtime):
|
||||
blocks = [nn.Linear(2, 2), nn.Linear(2, 2)]
|
||||
expected = []
|
||||
for index, block in enumerate(blocks):
|
||||
block.weight.data.fill_(index + 1)
|
||||
block.bias.data.fill_(index + 3)
|
||||
expected.append((block.weight.detach().clone(), block.bias.detach().clone()))
|
||||
|
||||
group = PinnedResidentLayerGroup(
|
||||
blocks,
|
||||
device=torch.device("cpu"),
|
||||
copy_stream=DummyStream(),
|
||||
pin_memory=False,
|
||||
)
|
||||
|
||||
assert all(block.weight.numel() == 0 for block in blocks)
|
||||
|
||||
group.load()
|
||||
for block, (weight, bias) in zip(blocks, expected):
|
||||
assert torch.equal(block.weight, weight)
|
||||
assert torch.equal(block.bias, bias)
|
||||
|
||||
# Device-side changes must not overwrite the persistent host master.
|
||||
blocks[0].weight.data.zero_()
|
||||
group.offload()
|
||||
assert all(block.weight.numel() == 0 for block in blocks)
|
||||
|
||||
group.load()
|
||||
for block, (weight, bias) in zip(blocks, expected):
|
||||
assert torch.equal(block.weight, weight)
|
||||
assert torch.equal(block.bias, bias)
|
||||
|
||||
group.offload()
|
||||
|
||||
def test_mmap_loader_attrs_survive_to_empty_parameter_replacement(self):
|
||||
module = nn.Linear(2, 2, bias=False)
|
||||
|
||||
def transform(tensor):
|
||||
return tensor.flip(0)
|
||||
|
||||
module.weight.mmap_weight_transform = transform
|
||||
|
||||
backend = object.__new__(DistributedLayerwiseOffloadBackend)
|
||||
backend._remember_mmap_param_attrs(module)
|
||||
module.to_empty(device="meta")
|
||||
|
||||
replacement = nn.Parameter(torch.arange(4).reshape(2, 2).float())
|
||||
backend._attach_mmap_param_attrs("weight", replacement, module.weight)
|
||||
|
||||
assert replacement.mmap_weight_transform is transform
|
||||
assert replacement.mmap_weight_transform_pending is True
|
||||
|
||||
def test_shared_allgather_output_is_narrowed_to_current_block(
|
||||
self, dist_group, patched_offload_runtime, monkeypatch
|
||||
):
|
||||
block = nn.Module()
|
||||
block.weight = nn.Parameter(torch.zeros(3, dtype=torch.float32))
|
||||
next_block = nn.Module()
|
||||
next_block.weight = nn.Parameter(torch.arange(3, dtype=torch.float32))
|
||||
|
||||
hook = DistributedLayerwiseOffloadHook(
|
||||
next_block=next_block,
|
||||
device=torch.device("cpu"),
|
||||
dp_group=object(),
|
||||
dp_size=2,
|
||||
rank=0,
|
||||
pin_memory=False,
|
||||
)
|
||||
hook.initialize_hook(block)
|
||||
# Simulate unified buffers sized for a larger heterogeneous block.
|
||||
hook.gpu_buffers = [
|
||||
{torch.float32: torch.empty(12)},
|
||||
{torch.float32: torch.empty(12)},
|
||||
]
|
||||
hook.gpu_shard_buffers = [
|
||||
{torch.float32: torch.empty(6)},
|
||||
{torch.float32: torch.empty(6)},
|
||||
]
|
||||
output_sizes = []
|
||||
|
||||
def fake_allgather(output, local_shard, *, group):
|
||||
del group
|
||||
output_sizes.append(output.numel())
|
||||
output.zero_()
|
||||
output[: local_shard.numel()].copy_(local_shard)
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "all_gather_into_tensor", fake_allgather)
|
||||
|
||||
hook.prefetch_layer(slot=0, non_blocking=False)
|
||||
|
||||
assert output_sizes == [4]
|
||||
|
||||
|
||||
class TestPinnedModuleStager:
|
||||
def test_offload_rebinds_immutable_cpu_master(self, patched_offload_runtime):
|
||||
module = nn.Linear(2, 2)
|
||||
expected_weight = module.weight.detach().clone()
|
||||
expected_bias = module.bias.detach().clone()
|
||||
stager = PinnedModuleStager(
|
||||
module,
|
||||
torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
copy_stream=DummyStream(),
|
||||
)
|
||||
|
||||
stager.load()
|
||||
module.weight.data.zero_()
|
||||
module.bias.data.zero_()
|
||||
stager.offload()
|
||||
|
||||
assert torch.equal(module.weight, expected_weight)
|
||||
assert torch.equal(module.bias, expected_bias)
|
||||
stager.load()
|
||||
assert torch.equal(module.weight, expected_weight)
|
||||
assert torch.equal(module.bias, expected_bias)
|
||||
stager.offload()
|
||||
|
||||
|
||||
class _DummyBlock(nn.Module):
|
||||
def __init__(self):
|
||||
@@ -504,6 +665,81 @@ class TestCrossGroupSharedBuffer:
|
||||
assert not sync_called[0], "Should skip sync-prefetch when slot_group matches (non-contaminated)"
|
||||
|
||||
|
||||
class TestAllGatherOutputSlicing:
|
||||
"""Regression test: with heterogeneous block sizes (e.g. MiniMax H3's
|
||||
token_refiner with blocks of ~1231MB and ~239MB), the shared AllGather
|
||||
output buffers are sized to the *largest* block across all groups.
|
||||
prefetch_layer must slice the output buffer down to each block's actual
|
||||
AllGather output size, otherwise output.numel() != dp_size * input.numel()
|
||||
and all_gather_into_tensor raises during enable()'s first prefetch.
|
||||
"""
|
||||
|
||||
def test_prefetch_slices_shared_output_buffer_to_block_size(self, dist_group, patched_offload_runtime, monkeypatch):
|
||||
dp_size = 2
|
||||
|
||||
# Heterogeneous block sizes: 8 vs 32 elements.
|
||||
small_block = nn.Module()
|
||||
small_block.weight = nn.Parameter(torch.arange(8, dtype=torch.float32))
|
||||
large_block = nn.Module()
|
||||
large_block.weight = nn.Parameter(torch.arange(100, 132, dtype=torch.float32))
|
||||
|
||||
def make_hook(next_block: nn.Module) -> DistributedLayerwiseOffloadHook:
|
||||
hook = DistributedLayerwiseOffloadHook(
|
||||
next_block=next_block,
|
||||
device=torch.device("cpu"),
|
||||
dp_group=dist.group.WORLD, # non-None -> AllGather path (mocked below)
|
||||
dp_size=dp_size,
|
||||
rank=0,
|
||||
pin_memory=False,
|
||||
)
|
||||
current_block = nn.Module()
|
||||
current_block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
|
||||
hook.initialize_hook(current_block)
|
||||
return hook
|
||||
|
||||
hook_small = make_hook(small_block)
|
||||
hook_large = make_hook(large_block)
|
||||
hooks = [hook_small, hook_large]
|
||||
|
||||
# Assign max-sized shared buffers exactly as backend.enable() does.
|
||||
shared_buffers = DistributedLayerwiseOffloadBackend._allocate_shared_buffers(hooks)
|
||||
shared_shard_buffers = DistributedLayerwiseOffloadBackend._allocate_shared_shard_buffers(hooks)
|
||||
for hook in hooks:
|
||||
hook.gpu_buffers = shared_buffers
|
||||
hook.gpu_shard_buffers = shared_shard_buffers
|
||||
|
||||
# Shared output buffer must be sized to the largest block (32).
|
||||
assert shared_buffers[0][torch.float32].numel() == 32
|
||||
|
||||
# Fake AllGather that enforces the real torch size contract and
|
||||
# emulates reconstruction of the full block from all ranks' shards.
|
||||
full_values = {
|
||||
8: torch.arange(8, dtype=torch.float32),
|
||||
32: torch.arange(100, 132, dtype=torch.float32),
|
||||
}
|
||||
ag_output_numels: list[int] = []
|
||||
|
||||
def fake_all_gather(output, input, group=None):
|
||||
assert output.numel() == dp_size * input.numel(), (
|
||||
f"AllGather contract violated: output.numel()={output.numel()} "
|
||||
f"!= {dp_size} * input.numel()={input.numel()}"
|
||||
)
|
||||
ag_output_numels.append(output.numel())
|
||||
output.copy_(full_values[output.numel()])
|
||||
|
||||
monkeypatch.setattr(dist, "all_gather_into_tensor", fake_all_gather)
|
||||
|
||||
# Small block: output must be sliced to dp_size * shard = 8, not 32.
|
||||
hook_small.prefetch_layer(slot=0, non_blocking=False)
|
||||
assert ag_output_numels[-1] == 8
|
||||
assert torch.equal(small_block.weight, full_values[8])
|
||||
|
||||
# Large block: uses the full max-sized buffer.
|
||||
hook_large.prefetch_layer(slot=1, non_blocking=False)
|
||||
assert ag_output_numels[-1] == 32
|
||||
assert torch.equal(large_block.weight, full_values[32])
|
||||
|
||||
|
||||
class TestOffloadPlan:
|
||||
"""Test declarative OffloadPlan metadata."""
|
||||
|
||||
@@ -517,6 +753,7 @@ class TestOffloadPlan:
|
||||
plan = OffloadPlan(
|
||||
block_attrs={"transformer": ("gen_layers",)},
|
||||
offload_submodules={"context_encoder": "layers"},
|
||||
resident_dit_paths=frozenset({"transformer"}),
|
||||
)
|
||||
|
||||
class PipelineWithPlan(nn.Module):
|
||||
@@ -527,12 +764,14 @@ class TestOffloadPlan:
|
||||
assert result is plan
|
||||
assert result.block_attrs == {"transformer": ("gen_layers",)}
|
||||
assert result.offload_submodules == {"context_encoder": "layers"}
|
||||
assert result.resident_dit_paths == frozenset({"transformer"})
|
||||
|
||||
def test_offload_plan_defaults_to_empty(self):
|
||||
"""OffloadPlan with no arguments should have empty dicts."""
|
||||
plan = OffloadPlan()
|
||||
assert plan.block_attrs == {}
|
||||
assert plan.offload_submodules == {}
|
||||
assert plan.resident_dit_paths == frozenset()
|
||||
|
||||
def test_offload_plan_is_frozen(self):
|
||||
"""OffloadPlan should be immutable (frozen=True)."""
|
||||
@@ -540,6 +779,50 @@ class TestOffloadPlan:
|
||||
with pytest.raises(Exception):
|
||||
plan.block_attrs = {"x": ("y",)} # type: ignore
|
||||
|
||||
def test_all_resident_dit_still_prepares_planned_submodules(self, patched_offload_runtime):
|
||||
class TokenRefiner(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.blocks = nn.ModuleList([nn.Linear(2, 2), nn.Linear(2, 2)])
|
||||
|
||||
class Transformer(nn.Module):
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.blocks = nn.ModuleList([nn.Linear(2, 2), nn.Linear(2, 2)])
|
||||
self.token_refiner = TokenRefiner()
|
||||
|
||||
class Pipeline(nn.Module):
|
||||
_offload_plan = OffloadPlan(
|
||||
offload_submodules={"token_refiner": "blocks"},
|
||||
resident_dit_paths=frozenset({"transformer"}),
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.transformer = Transformer()
|
||||
|
||||
pipeline = Pipeline()
|
||||
backend = DistributedLayerwiseOffloadBackend(
|
||||
OffloadConfig(
|
||||
strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE,
|
||||
pin_cpu_memory=False,
|
||||
dlo_use_allgather=False,
|
||||
dlo_resident_layers=2,
|
||||
),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
backend.enable(pipeline)
|
||||
|
||||
assert len(backend._resident_blocks) == 2
|
||||
assert len(backend._all_hook_groups) == 1
|
||||
assert len(backend._all_hook_groups[0]) == 2
|
||||
assert backend.enabled
|
||||
|
||||
backend.disable()
|
||||
|
||||
|
||||
class TestMmapValidation:
|
||||
"""Tests for mmap loader validation: TP rejection, strict check, validate_loaded_weights."""
|
||||
@@ -746,7 +1029,7 @@ class TestConfigValidation:
|
||||
OffloadConfig.from_od_config(FakeODConfig())
|
||||
|
||||
def test_hsdp_without_allgather_allowed(self):
|
||||
"""HSDP + DLO + no-AllGather should be allowed (full weights per rank)."""
|
||||
"""HSDP + DLO + no-AllGather uses standard-loader rank-local weights."""
|
||||
from vllm_omni.diffusion.offloader.base import OffloadConfig
|
||||
|
||||
class FakePC:
|
||||
@@ -768,12 +1051,33 @@ class TestConfigValidation:
|
||||
enable_layerwise_offload = False
|
||||
enable_distributed_layerwise_offload = True
|
||||
dlo_use_allgather = False # no AllGather → should be allowed
|
||||
dlo_resident_layers = 20
|
||||
pin_cpu_memory = True
|
||||
parallel_config = FakePC()
|
||||
model = "/fake/path"
|
||||
|
||||
config = OffloadConfig.from_od_config(FakeODConfig())
|
||||
assert config.dp_size == 1 # forced to 1 when no AllGather
|
||||
assert config.dlo_resident_layers == 20
|
||||
|
||||
def test_resident_layers_with_allgather_rejected(self):
|
||||
class FakePC:
|
||||
data_parallel_size = 2
|
||||
use_hsdp = False
|
||||
sequence_parallel_size = 1
|
||||
|
||||
class FakeODConfig:
|
||||
enable_cpu_offload = False
|
||||
enable_layerwise_offload = False
|
||||
enable_distributed_layerwise_offload = True
|
||||
dlo_use_allgather = True
|
||||
dlo_resident_layers = 20
|
||||
pin_cpu_memory = True
|
||||
parallel_config = FakePC()
|
||||
model = "/fake/path"
|
||||
|
||||
with pytest.raises(ValueError, match="requires --dlo-no-use-allgather"):
|
||||
OffloadConfig.from_od_config(FakeODConfig())
|
||||
|
||||
def test_num_inference_steps_none_rejected(self):
|
||||
"""DP multi-concurrency should reject None num_inference_steps."""
|
||||
|
||||
@@ -147,6 +147,130 @@ def test_get_npu_quant_method(mocker: MockerFixture, monkeypatch: pytest.MonkeyP
|
||||
assert isinstance(method, UnquantizedLinearMethod)
|
||||
|
||||
|
||||
def test_fused_layer_can_be_ignored_by_its_prefix(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
|
||||
"""A layer opts out of quantization under the prefix its module was built with.
|
||||
|
||||
MiniMax H3 stores qkv as one checkpoint tensor, so the fused name is the
|
||||
only handle a user has on it.
|
||||
"""
|
||||
from vllm_omni.quantization.int8_config import NPUInt8OnlineLinearMethod
|
||||
|
||||
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: False)
|
||||
monkeypatch.setattr(current_omni_platform, "is_npu", lambda: True)
|
||||
config = build_quant_config("int8", ignored_layers=["blocks.0.attn.qkv_proj"])
|
||||
layer = mocker.Mock(spec=LinearBase)
|
||||
|
||||
assert isinstance(config.get_quant_method(layer, "blocks.0.attn.qkv_proj"), UnquantizedLinearMethod)
|
||||
assert isinstance(config.get_quant_method(layer, "blocks.1.attn.qkv_proj"), NPUInt8OnlineLinearMethod)
|
||||
assert isinstance(config.get_quant_method(layer, "blocks.0.mlp.fc1"), NPUInt8OnlineLinearMethod)
|
||||
|
||||
|
||||
class TestNPUQuantMatmulShapeFallback:
|
||||
"""npu_quant_matmul rejects wide outputs, so those layers stay unquantized.
|
||||
|
||||
The limit applies to the per-rank shard, which is what the kernel sees. The
|
||||
same layer can therefore be quantized at a higher TP degree.
|
||||
"""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _mock_tp(self, mocker):
|
||||
# The fallback delegates to UnquantizedLinearMethod, which reads the TP
|
||||
# group; stand one in so the shape gating can be tested without a
|
||||
# distributed init.
|
||||
mock_group = mocker.Mock()
|
||||
mock_group.rank_in_group = 0
|
||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
|
||||
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", return_value=0)
|
||||
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
|
||||
|
||||
@pytest.fixture
|
||||
def quant_config(self):
|
||||
from vllm_omni.quantization.int8_config import DiffusionInt8Config
|
||||
|
||||
return DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
|
||||
|
||||
def create_weights(self, method, layer, output_partition_sizes):
|
||||
# Narrow input dim: the fallback path allocates for real, and only the
|
||||
# output dimension is under test.
|
||||
method.create_weights(
|
||||
layer,
|
||||
input_size_per_partition=8,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=8,
|
||||
output_size=sum(output_partition_sizes),
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda param, loaded_weight, *args, **kwargs: None,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method_name",
|
||||
["NPUInt8LinearMethod", "NPUInt8OnlineLinearMethod"],
|
||||
)
|
||||
def test_over_wide_layer_falls_back_to_unquantized(self, quant_config, method_name):
|
||||
import vllm_omni.quantization.int8_config as int8_config
|
||||
|
||||
method = getattr(int8_config, method_name)(quant_config)
|
||||
layer = Module()
|
||||
layer.quant_method = method
|
||||
|
||||
self.create_weights(method, layer, [int8_config.NPU_QUANT_MATMUL_MAX_OUT_FEATURES + 1])
|
||||
|
||||
assert isinstance(layer.quant_method, UnquantizedLinearMethod)
|
||||
assert layer.weight.dtype == torch.bfloat16
|
||||
|
||||
def test_layer_within_the_limit_keeps_int8(self, quant_config):
|
||||
from vllm_omni.quantization.int8_config import (
|
||||
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
|
||||
NPUInt8OnlineLinearMethod,
|
||||
)
|
||||
|
||||
method = NPUInt8OnlineLinearMethod(quant_config)
|
||||
layer = Module()
|
||||
layer.quant_method = method
|
||||
|
||||
self.create_weights(method, layer, [NPU_QUANT_MATMUL_MAX_OUT_FEATURES])
|
||||
|
||||
assert layer.quant_method is method
|
||||
|
||||
def test_tensor_parallel_sharding_brings_a_wide_layer_back(self, quant_config):
|
||||
from vllm_omni.quantization.int8_config import (
|
||||
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
|
||||
NPUInt8OnlineLinearMethod,
|
||||
)
|
||||
|
||||
global_out_features = 4 * NPU_QUANT_MATMUL_MAX_OUT_FEATURES
|
||||
method = NPUInt8OnlineLinearMethod(quant_config)
|
||||
layer = Module()
|
||||
layer.quant_method = method
|
||||
|
||||
self.create_weights(method, layer, [global_out_features // 4])
|
||||
|
||||
assert layer.quant_method is method
|
||||
|
||||
|
||||
class TestOffloadAfterQuant:
|
||||
def test_only_methods_advertising_the_capability_are_asked(self):
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.quantization.int8_config import DiffusionInt8Config, NPUInt8OnlineLinearMethod
|
||||
|
||||
class MetaDeviceOnlyMethod:
|
||||
"""Stands in for upstream online FP8: lazy, but no offload hook."""
|
||||
|
||||
uses_meta_device = True
|
||||
|
||||
model = torch.nn.Sequential(Module(), Module())
|
||||
lazy_int8 = NPUInt8OnlineLinearMethod(
|
||||
DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
|
||||
)
|
||||
model[0].quant_method = lazy_int8
|
||||
model[1].quant_method = MetaDeviceOnlyMethod()
|
||||
|
||||
marked = DiffusersPipelineLoader._request_offload_after_quant(model)
|
||||
|
||||
assert marked == 1
|
||||
assert lazy_int8._offload_after_quant
|
||||
|
||||
|
||||
class TestInt8LinearMethod:
|
||||
@pytest.fixture
|
||||
def mock_quant_config(self, mocker):
|
||||
|
||||
@@ -131,6 +131,11 @@ def test_qwen_image_edit_plus_sets_generic_multimodal_limit():
|
||||
assert od_config.max_multimodal_image_inputs == QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES
|
||||
|
||||
|
||||
def test_task_type_roundtrip():
|
||||
od = _roundtrip_diffusion_config(model="x", task_type="model-defined-task")
|
||||
assert od.task_type == "model-defined-task"
|
||||
|
||||
|
||||
def test_additional_config_roundtrip():
|
||||
additional_config = {"torchair_graph_config": {"enabled": True}}
|
||||
od = _roundtrip_diffusion_config(model="x", additional_config=additional_config)
|
||||
|
||||
@@ -247,6 +247,56 @@ class TestRequestModeDispatch:
|
||||
results = {ro.request_id: ro.result.error for ro in out.runner_outputs}
|
||||
assert results == {request_id: f"result_for_{request_id}" for request_id in request_ids}
|
||||
|
||||
def test_dlo_dp_routes_multiple_requests_without_pipeline_batching(self):
|
||||
executor, _, _ = _make_executor(num_gpus=2)
|
||||
executor.od_config = SimpleNamespace(
|
||||
step_execution=False,
|
||||
parallel_config=SimpleNamespace(data_parallel_size=2),
|
||||
enable_distributed_layerwise_offload=True,
|
||||
dlo_use_allgather=True,
|
||||
)
|
||||
executor.execute_request = Mock(return_value="dlo-dp")
|
||||
executor.collective_rpc = Mock()
|
||||
scheduler_output = _make_sched_output("A", "B")
|
||||
|
||||
result = executor.execute_batch(scheduler_output)
|
||||
|
||||
assert result == "dlo-dp"
|
||||
executor.execute_request.assert_called_once_with(scheduler_output)
|
||||
executor.collective_rpc.assert_not_called()
|
||||
|
||||
def test_dlo_dp_multi_rank_reply_uses_synchronous_rpc_collection(self):
|
||||
executor, req_q, res_q = _make_executor(num_gpus=2)
|
||||
executor.od_config = SimpleNamespace(
|
||||
step_execution=False,
|
||||
parallel_config=SimpleNamespace(data_parallel_size=2),
|
||||
)
|
||||
executor._sync_result_buffer = res_q
|
||||
|
||||
def worker():
|
||||
request = req_q.get(timeout=2)
|
||||
assert "rpc_id" not in request
|
||||
wave_id = request["wave_id"]
|
||||
for dp_rank in range(2):
|
||||
res_q.put(
|
||||
{
|
||||
"dp_rank": dp_rank,
|
||||
"output": _tagged_output(str(dp_rank)),
|
||||
"wave_id": wave_id,
|
||||
}
|
||||
)
|
||||
|
||||
thread = threading.Thread(target=worker, daemon=True)
|
||||
thread.start()
|
||||
results = executor.collective_rpc(
|
||||
"execute_model",
|
||||
unique_reply_rank=None,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
thread.join(timeout=2)
|
||||
|
||||
assert [result.error for result in results] == ["0", "1"]
|
||||
|
||||
|
||||
# ───────────────── concurrent collective RPC ─────────────────
|
||||
|
||||
|
||||
Reference in new issue
Block a user