[Feat][OmniVoice]Support Varlen Attn, Request-Batch and Step-Execution (#6408)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
This commit is contained in:
boatman authored and GitHub committed 2026-09-08 18:58:38 -07:00
1 parent d7678e2bb3
commit cdf027874c
14 files changed
+1396 -610

No files matched your search

@@ -6,19 +6,27 @@ Tests for OmniVoice CUDA Graph generator wrapper numerical equivalence.
Verifies that _OmniVoiceCUDAGraphForward produces results equivalent to
eager mode across three scenarios:
- Exact-size inputs (no padding) → bit-identical
- Padded inputs (padded to nearest bucket) → correct slicing, exact match
for position-independent models
- Oversized inputs (fallback to eager) → bit-identical
- Padded inputs (padded to nearest bucket) → correct slicing and exact match
- Oversized inputs (128-aligned lazy capture) → bit-identical
Uses a lightweight synthetic generator to keep warmup fast and avoid
loading actual model weights.
Uses a small randomly initialized OmniVoiceGenerator so every test exercises
the production embedding, transformer, varlen-attention, and logits paths
without loading checkpoint weights.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.utils.math_utils import round_up
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
OmniVoiceGenerator,
_OmniVoiceCUDAGraphForward,
)
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
pytestmark = [
pytest.mark.core_model,
@@ -29,53 +37,7 @@ pytestmark = [
DEVICE = torch.device("cuda:0")
NUM_CB = 8
VOCAB = 1025
HIDDEN = 64
CAPTURE_SIZES = [32, 64, 128]
HEAD_DIM = 64
# ---------------------------------------------------------------------------
# Synthetic generator: matches _OmniVoiceCUDAGraphForward's interface
# ---------------------------------------------------------------------------
class _SyntheticGenerator:
"""Position-independent Embedding+Linear replacing OmniVoiceGenerator's 28-layer transformer.
Each token position is processed independently (no attention), so padded
runs produce bit-identical results to non-padded runs on the same positions.
This makes the padding/slicing logic easy to verify exactly.
"""
class _Cfg:
num_audio_codebook = NUM_CB
def __init__(self, device: torch.device):
self.config = self._Cfg()
self.text_embedding = nn.Embedding(1000, HIDDEN).to(device).eval()
self._linear = nn.Linear(HIDDEN, NUM_CB * VOCAB, bias=False).to(device).eval()
self._rope_table: torch.Tensor | None = None
@property
def model_dtype(self) -> torch.dtype:
"""Part of the interface _OmniVoiceCUDAGraphForward expects of a generator."""
return self.text_embedding.weight.dtype
def _rope_table_for(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
"""Part of the interface _OmniVoiceCUDAGraphForward expects of a generator."""
return torch.zeros(seq_len, HEAD_DIM, device=device, dtype=dtype)
def _step_forward(
self,
input_ids: torch.Tensor,
audio_mask: torch.Tensor,
attention_mask: torch.Tensor | None,
rope_table: torch.Tensor,
) -> torch.Tensor:
two_b, _, S = input_ids.shape
x = self.text_embedding(input_ids[:, 0, :].clamp(0, 999)) # [two_b, S, H]
logits = self._linear(x) # [two_b, S, 8*1025]
return logits.view(two_b, S, NUM_CB, VOCAB).permute(0, 2, 1, 3) # [two_b, 8, S, 1025]
# ---------------------------------------------------------------------------
@@ -83,18 +45,37 @@ class _SyntheticGenerator:
# ---------------------------------------------------------------------------
def _eager(gen: _SyntheticGenerator, ids: torch.Tensor, mask: torch.Tensor, attn: torch.Tensor) -> torch.Tensor:
"""Run _step_forward with a zero RoPE table (synthetic gen ignores it)."""
two_b, _, S = ids.shape
return gen._step_forward(ids, mask, attn, gen._rope_table_for(S, ids.device, torch.float32))
def _eager(gen: OmniVoiceGenerator, ids: torch.Tensor, mask: torch.Tensor, cu_seqs) -> torch.Tensor:
"""Run the production varlen step outside CUDA Graph."""
seq_len = ids.shape[0]
rope_table = gen._rope_table_for(seq_len, ids.device, gen.model_dtype)
return gen._step_forward(ids, mask, cu_seqs, rope_table)
def _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, batch_size) -> torch.Tensor:
"""Run eager with the exact padded shape and metadata used by Graph."""
seq_len = ids.shape[0]
bucket = wrapper._find_bucket(batch_size, seq_len)
if bucket is None:
bucket = round_up(seq_len, wrapper._LAZY_CAPTURE_ALIGNMENT)
padded_ids, padded_mask = wrapper._pad_inputs(ids, mask, bucket)
padded_cu_seqs = cu_seqs.clone()
padded_cu_seqs[-1] = bucket
return _eager(gen, padded_ids, padded_mask, padded_cu_seqs)[:, :seq_len, :]
def _make_inputs(seq_len: int, device: torch.device = DEVICE):
two_b = 2
ids = torch.randint(0, 100, (two_b, NUM_CB, seq_len), dtype=torch.long, device=device)
mask = torch.ones(two_b, seq_len, dtype=torch.bool, device=device)
attn = torch.ones(two_b, 1, seq_len, seq_len, dtype=torch.bool, device=device)
return ids, mask, attn
ids = torch.randint(0, 100, (seq_len, NUM_CB), dtype=torch.long, device=device)
mask = torch.ones(seq_len, dtype=torch.bool, device=device)
cond_len = (seq_len + 1) // 2
uncond_len = seq_len - cond_len
cu_seqs = torch.tensor(
[0, cond_len, cond_len + uncond_len, cond_len + uncond_len],
dtype=torch.int32,
device=device,
)
return ids, mask, cu_seqs
# ---------------------------------------------------------------------------
@@ -105,15 +86,26 @@ def _make_inputs(seq_len: int, device: torch.device = DEVICE):
@pytest.fixture(scope="module")
def gen():
torch.manual_seed(42)
return _SyntheticGenerator(DEVICE)
config = OmniVoiceConfig(
audio_vocab_size=VOCAB,
num_audio_codebook=NUM_CB,
llm_config={
"hidden_size": 64,
"num_hidden_layers": 2,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"intermediate_size": 128,
"vocab_size": 128,
"max_position_embeddings": 4096,
"head_dim": 16,
},
enable_cuda_graph=False,
)
return OmniVoiceGenerator(config, SimpleNamespace(max_num_seqs=2)).to(DEVICE).eval()
@pytest.fixture(scope="module")
def wrapper(gen):
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
_OmniVoiceCUDAGraphForward,
)
w = _OmniVoiceCUDAGraphForward(gen, capture_sizes=CAPTURE_SIZES)
w.warmup(DEVICE)
return w
@@ -127,10 +119,10 @@ def wrapper(gen):
@pytest.mark.parametrize("seq_len", CAPTURE_SIZES)
def test_exact_size_bit_identical(gen, wrapper, seq_len):
"""When input exactly matches a captured bucket, output must be bit-identical to eager."""
ids, mask, attn = _make_inputs(seq_len)
ids, mask, cu_seqs = _make_inputs(seq_len)
with torch.no_grad():
eager_out = _eager(gen, ids, mask, attn)
graph_out = wrapper(ids, mask, attn)
eager_out = _eager(gen, ids, mask, cu_seqs)
graph_out = wrapper(ids, mask, cu_seqs, 1)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@@ -140,40 +132,36 @@ def test_exact_size_bit_identical(gen, wrapper, seq_len):
@pytest.mark.parametrize("seq_len", [1, 15, 33, 60, 100])
def test_padded_output_shape(gen, wrapper, seq_len):
def test_padded_output_shape(wrapper, seq_len):
"""Graph output must be sliced back to actual seq_len, not the bucket size."""
ids, mask, attn = _make_inputs(seq_len)
ids, mask, cu_seqs = _make_inputs(seq_len)
with torch.no_grad():
graph_out = wrapper(ids, mask, attn)
assert graph_out.shape == (2, NUM_CB, seq_len, VOCAB)
graph_out = wrapper(ids, mask, cu_seqs, 1)
assert graph_out.shape == (NUM_CB, seq_len, VOCAB)
@pytest.mark.parametrize("seq_len", [15, 33, 60, 100])
def test_padded_output_matches_eager(gen, wrapper, seq_len):
"""Padded graph output must equal eager output at actual positions.
The synthetic model has no attention across positions, so zero-padding
does not affect non-padded positions — exact match is expected.
"""
ids, mask, attn = _make_inputs(seq_len)
"""Padded graph output must equal eager output at actual positions."""
ids, mask, cu_seqs = _make_inputs(seq_len)
with torch.no_grad():
eager_out = _eager(gen, ids, mask, attn)
graph_out = wrapper(ids, mask, attn)
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 1)
graph_out = wrapper(ids, mask, cu_seqs, 1)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
# ---------------------------------------------------------------------------
# 3. Oversized inputs → fallback to eager (lazy capture), bit-identical
# 3. Oversized inputs → aligned lazy capture, bit-identical
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("seq_len", [129, 200, 256])
def test_fallback_eager_bit_identical(gen, wrapper, seq_len):
"""Sequences exceeding the largest bucket fall back to lazy capture → bit-identical."""
ids, mask, attn = _make_inputs(seq_len)
def test_aligned_lazy_capture_bit_identical(gen, wrapper, seq_len):
"""Sequences beyond the static plan use aligned lazy capture and remain bit-identical."""
ids, mask, cu_seqs = _make_inputs(seq_len)
with torch.no_grad():
eager_out = _eager(gen, ids, mask, attn)
graph_out = wrapper(ids, mask, attn)
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 1)
graph_out = wrapper(ids, mask, cu_seqs, 1)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@@ -184,13 +172,73 @@ def test_fallback_eager_bit_identical(gen, wrapper, seq_len):
def test_deterministic_across_calls(wrapper):
"""Same input must produce identical output on repeated CUDA graph replays."""
ids, mask, attn = _make_inputs(32)
ids, mask, cu_seqs = _make_inputs(32)
with torch.no_grad():
out1 = wrapper(ids, mask, attn)
out2 = wrapper(ids, mask, attn)
out1 = wrapper(ids, mask, cu_seqs, 1).clone()
out2 = wrapper(ids, mask, cu_seqs, 1).clone()
torch.testing.assert_close(out1, out2, atol=0, rtol=0)
def test_replay_uses_updated_cu_seqs(gen, wrapper):
"""The same graph key must honor new sequence boundaries on replay."""
ids, mask, _ = _make_inputs(48)
cu_seqs_a = torch.tensor([0, 32, 48, 48], dtype=torch.int32, device=DEVICE)
cu_seqs_b = torch.tensor([0, 17, 48, 48], dtype=torch.int32, device=DEVICE)
with torch.no_grad():
eager_a = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs_a, 1)
graph_a = wrapper(ids, mask, cu_seqs_a, 1).clone()
eager_b = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs_b, 1)
graph_b = wrapper(ids, mask, cu_seqs_b, 1).clone()
torch.testing.assert_close(graph_a, eager_a, atol=0, rtol=0)
torch.testing.assert_close(graph_b, eager_b, atol=0, rtol=0)
assert not torch.equal(graph_a, graph_b)
def test_batch_two_cu_seqs_matches_eager(gen, wrapper):
"""B=2 uses four real sequence segments plus the fixed tail slot."""
ids, mask, _ = _make_inputs(48)
cu_seqs = torch.tensor([0, 10, 18, 32, 48, 48], dtype=torch.int32, device=DEVICE)
assert cu_seqs.numel() == 2 * 2 + 2
with torch.no_grad():
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 2)
graph_out = wrapper(ids, mask, cu_seqs, 2).clone()
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
def test_lazy_graph_cache_uses_lru_eviction(wrapper):
"""A lazy-cache hit must protect that graph from the next eviction."""
original_limit = wrapper._MAX_LAZY_GRAPHS
wrapper._lazy_graphs.clear()
wrapper._MAX_LAZY_GRAPHS = 2
try:
# Static B=1 coverage ends at 128. These lengths round to three
# distinct 128-aligned lazy keys: 256, 384, and 512.
for seq_len in (129, 257):
ids, mask, cu_seqs = _make_inputs(seq_len)
with torch.no_grad():
wrapper(ids, mask, cu_seqs, 1)
assert list(wrapper._lazy_graphs) == [(1, 256), (1, 384)]
# Refresh key 256, making key 384 the least recently used entry.
ids, mask, cu_seqs = _make_inputs(129)
with torch.no_grad():
wrapper(ids, mask, cu_seqs, 1)
assert list(wrapper._lazy_graphs) == [(1, 384), (1, 256)]
# Inserting key 512 must evict key 384, not the recently hit key 256.
ids, mask, cu_seqs = _make_inputs(385)
with torch.no_grad():
wrapper(ids, mask, cu_seqs, 1)
assert list(wrapper._lazy_graphs) == [(1, 256), (1, 512)]
finally:
wrapper._MAX_LAZY_GRAPHS = original_limit
wrapper._lazy_graphs.clear()
# ---------------------------------------------------------------------------
# 5. _find_bucket logic (CPU, no CUDA graph)
# ---------------------------------------------------------------------------
@@ -203,58 +251,17 @@ def test_find_bucket_returns_nearest_bucket():
)
w = _OmniVoiceCUDAGraphForward.__new__(_OmniVoiceCUDAGraphForward)
w._capture_sizes = [32, 64, 128]
w.capture_bucket_sizes_by_batch = {
1: [32, 64, 128],
2: [64, 128, 256],
}
w._graphs = {}
assert w._find_bucket(1) == 32
assert w._find_bucket(32) == 32
assert w._find_bucket(33) == 64
assert w._find_bucket(64) == 64
assert w._find_bucket(128) == 128
assert w._find_bucket(129) is None
assert w._find_bucket(1000) is None
# ---------------------------------------------------------------------------
# 6. enable_cuda_graph=False produces same tokens as enable_cuda_graph=True
# ---------------------------------------------------------------------------
def test_cuda_graph_disabled_matches_eager_generator():
"""OmniVoiceGenerator with enable_cuda_graph=False must produce the same
_step_forward output as one with enable_cuda_graph=True (after warmup).
Uses a 2-layer config so warmup completes quickly without loading weights.
"""
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import OmniVoiceGenerator
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
cfg = OmniVoiceConfig()
cfg.llm_num_hidden_layers = 2
cfg.num_hidden_layers = 2
torch.manual_seed(0)
gen_eager = OmniVoiceGenerator(cfg).to(DEVICE).eval()
gen_eager.config.enable_cuda_graph = False
gen_eager._cuda_graph_fwd = None
torch.manual_seed(0)
gen_graph = OmniVoiceGenerator(cfg).to(DEVICE).eval()
gen_graph.config.cuda_graph_capture_sizes = [64]
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import _OmniVoiceCUDAGraphForward
gen_graph._cuda_graph_fwd = _OmniVoiceCUDAGraphForward(gen_graph, capture_sizes=[64])
gen_graph._cuda_graph_fwd.warmup(DEVICE)
seq_len = 64
ids = torch.zeros(2, cfg.num_audio_codebook, seq_len, dtype=torch.long, device=DEVICE)
mask = torch.ones(2, seq_len, dtype=torch.bool, device=DEVICE)
attn = torch.ones(2, 1, seq_len, seq_len, dtype=torch.bool, device=DEVICE)
rope_table = gen_eager._rope_table_for(seq_len, DEVICE, gen_eager.model_dtype)
with torch.no_grad():
eager_logits = gen_eager._step_forward(ids, mask, attn, rope_table)
graph_logits = gen_graph._cuda_graph_fwd(ids, mask, attn)
torch.testing.assert_close(graph_logits, eager_logits, atol=0, rtol=0)
assert w._find_bucket(1, 1) == 32
assert w._find_bucket(1, 32) == 32
assert w._find_bucket(1, 33) == 64
assert w._find_bucket(1, 128) == 128
assert w._find_bucket(1, 129) is None
assert w._find_bucket(2, 33) == 64
assert w._find_bucket(2, 129) == 256
assert w._find_bucket(3, 64) is None
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The OmniVoice generator packs q/k/v and gate/up into fused projections.
The HF checkpoint stores those five tensors separately, so packing them is a
@@ -14,6 +14,8 @@ that would leave one partly written raises instead.
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
@@ -49,6 +51,13 @@ def _config() -> OmniVoiceConfig:
)
def _generator() -> OmniVoiceGenerator:
return OmniVoiceGenerator(
_config(),
SimpleNamespace(max_num_seqs=1),
)
def _checkpoint_shards() -> dict[str, torch.Tensor]:
"""The five per-layer tensors an OmniVoice checkpoint actually stores."""
torch.manual_seed(0)
@@ -64,7 +73,7 @@ def _checkpoint_shards() -> dict[str, torch.Tensor]:
def test_every_fused_parameter_is_written() -> None:
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = _checkpoint_shards()
loaded = generator._load_fused_projections(shards)
@@ -77,7 +86,7 @@ def test_every_fused_parameter_is_written() -> None:
def test_packed_weight_matches_the_shards_it_came_from() -> None:
"""Packing order is q,k,v and gate,up -- the split in forward() assumes it."""
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = _checkpoint_shards()
generator._load_fused_projections(shards)
@@ -101,7 +110,7 @@ def test_packed_weight_matches_the_shards_it_came_from() -> None:
def test_no_fused_parameter_keeps_its_random_init() -> None:
"""The corruption this guards against: a fused param never written at all."""
generator = OmniVoiceGenerator(_config())
generator = _generator()
before = {
name: param.detach().clone()
for name, param in generator.named_parameters()
@@ -121,7 +130,7 @@ def test_no_fused_parameter_keeps_its_random_init() -> None:
["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj", "mlp.gate_proj", "mlp.up_proj"],
)
def test_a_missing_shard_raises_instead_of_loading_partially(dropped: str) -> None:
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = _checkpoint_shards()
del shards[f"llm.layers.0.{dropped}.weight"]
@@ -130,7 +139,7 @@ def test_a_missing_shard_raises_instead_of_loading_partially(dropped: str) -> No
def test_a_wrong_shaped_shard_raises() -> None:
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = _checkpoint_shards()
shards["llm.layers.0.self_attn.q_proj.weight"] = torch.randn(NUM_HEADS * HEAD_DIM + 1, HIDDEN)
@@ -140,7 +149,7 @@ def test_a_wrong_shaped_shard_raises() -> None:
def test_a_checkpoint_missing_a_whole_layer_raises() -> None:
"""Half-loaded is the dangerous state: it neither errors nor works."""
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = {k: v for k, v in _checkpoint_shards().items() if not k.startswith("llm.layers.1.")}
with pytest.raises(ValueError, match="fused projections"):
@@ -149,7 +158,7 @@ def test_a_checkpoint_missing_a_whole_layer_raises() -> None:
def test_a_checkpoint_with_no_fused_shards_is_left_alone() -> None:
"""An unrelated state_dict must not trip the completeness check."""
generator = OmniVoiceGenerator(_config())
generator = _generator()
assert generator._load_fused_projections({"audio_heads.weight": torch.randn(4, 4)}) == set()
@@ -162,7 +171,7 @@ def test_load_weights_wires_the_packing_in(tmp_path, monkeypatch: pytest.MonkeyP
"""
import safetensors.torch
generator = OmniVoiceGenerator(_config())
generator = _generator()
shards = _checkpoint_shards()
before = generator.layers[0].self_attn.qkv_proj.weight.detach().clone()
@@ -1,183 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OmniVoice attention masks must follow the model dtype, not a hardcoded float32.
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""OmniVoice SDPA fallback masks must not depend on the model dtype.
SDPA requires the additive attention mask to carry the same dtype as the query.
The generator used to materialize that mask as float32 unconditionally, in three
places (the shared per-forward mask, the CUDA-graph capture buffer, and the
replay-time normalization), so every half-precision deployment died on the first
attention call with::
The previous padded attention path used an additive floating-point mask. SDPA
requires such a mask to have the same dtype as the query, and constructing it
as float32 caused half-precision OmniVoice inference to fail on CUDA.
RuntimeError: invalid dtype for bias - should match query's dtype
(the exact wording depends on which SDPA backend is selected for the shape; the
math backend words the same rejection as ``attn_mask.dtype``.)
That left OmniVoice servable only in float32, even though the upstream k2-fsa
implementation runs it in float16.
Note the split between the CPU and CUDA tests below: the CPU SDPA backend
silently accepts the mismatched mask, so only the CUDA tests can pin the actual
crash. The CPU tests cover the mask contract itself, which is what the fix
changes and what a future regression would break first.
The packed-varlen path no longer needs an additive mask. When SDPA fallback is
required, ``_attention_metadata_from_cu_seqs`` builds a boolean block mask.
Boolean SDPA masks have no dtype coupling with the query, so no mask coercion
is required for float16 or bfloat16 inference.
"""
from __future__ import annotations
import pytest
import torch
import torch.nn.functional as F
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
OmniVoiceAttention,
OmniVoiceGenerator,
_additive_float_mask,
_attention_metadata_from_cu_seqs,
)
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
HALF_DTYPES = [torch.float16, torch.bfloat16]
ALL_DTYPES = HALF_DTYPES + [torch.float32]
cpu_test = pytest.mark.core_model
cuda_test = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def _tiny_config() -> OmniVoiceConfig:
"""A real OmniVoiceConfig sized so the module fits in a unit test."""
return OmniVoiceConfig(
llm_config={
"hidden_size": 32,
"num_hidden_layers": 2,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"head_dim": 8,
"intermediate_size": 64,
"vocab_size": 64,
"max_position_embeddings": 128,
},
enable_cuda_graph=False,
@cpu_test
@pytest.mark.cpu
def test_sdpa_fallback_mask_is_dtype_independent() -> None:
"""The fallback mask is boolean rather than an additive model-dtype mask."""
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
metadata = _attention_metadata_from_cu_seqs(
cu_seqs,
6,
needs_sdpa_mask=True,
)
def _inputs(dtype: torch.dtype, device: torch.device) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hidden = torch.randn(2, 6, 32, device=device, dtype=dtype)
bool_mask = torch.ones(2, 1, 6, 6, dtype=torch.bool, device=device)
bool_mask[:, :, :, 4:] = False
rope_table = torch.zeros(6, 8, device=device, dtype=dtype)
rope_table[:, :4] = 1.0 # cos = 1, sin = 0: no rotation, so the test is about the mask
return hidden, bool_mask, rope_table
# --------------------------------------------------------------------------
# The mask contract (CPU)
# --------------------------------------------------------------------------
@cpu_test
@pytest.mark.cpu
@pytest.mark.parametrize("dtype", ALL_DTYPES)
def test_additive_float_mask_uses_requested_dtype(dtype: torch.dtype) -> None:
out = _additive_float_mask(torch.tensor([[True, False]]), dtype)
assert out.dtype == dtype
@cpu_test
@pytest.mark.cpu
def test_additive_float_mask_keeps_mask_semantics() -> None:
"""Attend positions stay at 0.0 and masked ones at -inf, in half precision too."""
out = _additive_float_mask(torch.tensor([[True, False]]), torch.float16)
assert out[0, 0].item() == 0.0
assert out[0, 1].item() == float("-inf")
@cpu_test
@pytest.mark.cpu
def test_additive_float_mask_requires_an_explicit_dtype() -> None:
"""The float32 default is the bug; keeping the parameter required is the fix."""
with pytest.raises(TypeError):
_additive_float_mask(torch.tensor([[True]])) # type: ignore[call-arg]
@cpu_test
@pytest.mark.cpu
@pytest.mark.parametrize("dtype", ALL_DTYPES)
def test_generator_model_dtype_tracks_its_weights(dtype: torch.dtype) -> None:
"""The single source of truth the three former float32 literals now defer to."""
assert OmniVoiceGenerator(_tiny_config()).to(dtype).model_dtype == dtype
@cpu_test
@pytest.mark.cpu
@pytest.mark.parametrize("dtype", HALF_DTYPES)
def test_attention_accepts_a_model_dtype_mask(dtype: torch.dtype) -> None:
attn = OmniVoiceAttention(_tiny_config()).to(dtype).eval()
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cpu"))
with torch.inference_mode():
out = attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, dtype))
assert out.shape == hidden.shape
assert out.dtype == dtype
assert torch.isfinite(out).all()
# --------------------------------------------------------------------------
# The crash itself (CUDA only — the CPU backend does not reject the mismatch)
# --------------------------------------------------------------------------
assert metadata.attn_mask is not None
assert metadata.attn_mask.dtype == torch.bool
@cuda_test
@pytest.mark.parametrize("dtype", HALF_DTYPES)
def test_float32_mask_is_what_broke_half_precision(dtype: torch.dtype) -> None:
"""Pin the original failure so a float32 default cannot come back unnoticed."""
attn = OmniVoiceAttention(_tiny_config()).to(device="cuda:0", dtype=dtype).eval()
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cuda:0"))
with pytest.raises(RuntimeError, match=r"(invalid dtype for bias|attn_mask)"), torch.inference_mode():
attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, torch.float32))
@cuda_test
@pytest.mark.parametrize("dtype", HALF_DTYPES)
def test_real_path_forward_runs_in_half_precision(dtype: torch.dtype) -> None:
"""The regression, on the production path with the Triton norm kernels live."""
attn = OmniVoiceAttention(_tiny_config()).to(device="cuda:0", dtype=dtype).eval()
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cuda:0"))
with torch.inference_mode():
out = attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, dtype))
assert out.dtype == dtype
assert torch.isfinite(out).all()
@cuda_test
@pytest.mark.parametrize("dtype", HALF_DTYPES)
def test_generator_forward_runs_in_half_precision(dtype: torch.dtype) -> None:
"""End-to-end over the real iterative loop, which is where the float32 mask was built.
This is the test that fails on an unfixed tree: ``forward`` materialized the
shared SDPA mask as float32 regardless of the weights' dtype.
"""
def test_sdpa_fallback_mask_needs_no_half_precision_coercion(
dtype: torch.dtype,
) -> None:
"""Half-precision SDPA accepts the production bool mask without casting."""
device = torch.device("cuda:0")
config = _tiny_config()
generator = OmniVoiceGenerator(config).to(device=device, dtype=dtype).eval()
seq_len = 6
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32, device=device)
seq_len, target_len = 12, 4
input_ids = torch.zeros(2, config.num_audio_codebook, seq_len, dtype=torch.long, device=device)
input_ids[:, 1:, :] = config.audio_mask_id
audio_mask = torch.zeros(2, seq_len, dtype=torch.bool, device=device)
audio_mask[:, seq_len - target_len :] = True
attention_mask = torch.ones(2, 1, seq_len, seq_len, dtype=torch.bool, device=device)
metadata = _attention_metadata_from_cu_seqs(
cu_seqs,
seq_len,
needs_sdpa_mask=True,
)
assert metadata.attn_mask is not None
assert metadata.attn_mask.dtype == torch.bool
query = torch.randn(1, 2, seq_len, 8, device=device, dtype=dtype)
key = torch.randn_like(query)
value = torch.randn_like(query)
with torch.inference_mode():
tokens = generator(
input_ids,
audio_mask,
attention_mask,
target_lens=[target_len],
seed=0,
num_step=2,
output = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=metadata.attn_mask,
)
assert tokens.shape == (1, config.num_audio_codebook, target_len)
assert tokens.dtype == torch.long
assert output.dtype == dtype
assert torch.isfinite(output).all()
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.models.omnivoice.pipeline_omnivoice import (
OmniVoicePipeline,
_PreparedOmniVoiceRequest,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
_attention_metadata_from_cu_seqs,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_sdpa_fallback_mask_preserves_packed_sequence_boundaries() -> None:
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
metadata = _attention_metadata_from_cu_seqs(cu_seqs, 6, needs_sdpa_mask=True)
assert metadata.attn_mask is not None
expected = torch.tensor(
[
[1, 1, 0, 0, 0, 0],
[1, 1, 0, 0, 0, 0],
[0, 0, 1, 0, 0, 0],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
],
dtype=torch.bool,
)
torch.testing.assert_close(metadata.attn_mask[0, 0], expected)
def test_eager_attention_metadata_honors_exact_max_seqlen() -> None:
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
metadata = _attention_metadata_from_cu_seqs(
cu_seqs,
6,
needs_sdpa_mask=False,
max_seqlen=3,
)
assert metadata.extra["max_seqlen_q"] == 3
assert metadata.extra["max_seqlen_k"] == 3
def test_request_batch_prepare_error_does_not_skip_valid_request() -> None:
"""A malformed request must not prevent another request from completing."""
prepared = _PreparedOmniVoiceRequest(
input_ids=torch.zeros(5, 8, dtype=torch.long),
audio_mask=torch.ones(5, dtype=torch.bool),
cond_len=3,
target_len=2,
seed=42,
)
def prepare_request(prompt, extra):
del extra
if prompt == "bad":
return DiffusionOutput(error="invalid prompt")
return prepared
def collate_requests(requests):
assert requests == [prepared]
return prepared.input_ids, prepared.audio_mask, [prepared.cond_len]
generator_calls = []
def generate_tokens(**kwargs):
generator_calls.append(kwargs)
assert kwargs["target_lens"] == [prepared.target_len]
return torch.ones(1, 8, prepared.target_len, dtype=torch.long)
pipeline = SimpleNamespace(
_prepare_request_input=prepare_request,
_collate_request_inputs=collate_requests,
generator=generate_tokens,
decoder=lambda tokens: tokens.float(),
num_step=32,
guidance_scale=7.0,
t_shift=1.0,
layer_penalty_factor=0.0,
position_temperature=0.0,
class_temperature=0.0,
)
sampling = OmniDiffusionSamplingParams(num_inference_steps=32, guidance_scale=7.0)
batch = DiffusionRequestBatch(
requests=[
OmniDiffusionRequest(prompt="bad", sampling_params=sampling, request_id="bad"),
OmniDiffusionRequest(
prompt="good",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=32, guidance_scale=7.0),
request_id="good",
),
]
)
outputs = OmniVoicePipeline.forward(pipeline, batch)
assert len(outputs) == 2
assert outputs[0].error == "invalid prompt"
assert outputs[1].error is None
torch.testing.assert_close(outputs[1].output, torch.ones(1, 8, 2))
assert generator_calls[0]["guidance_scale"] == 7.0
def test_request_batch_honors_explicit_sampling_overrides() -> None:
"""Request steps and guidance must override the OmniVoice defaults."""
prepared = _PreparedOmniVoiceRequest(
input_ids=torch.zeros(5, 8, dtype=torch.long),
audio_mask=torch.ones(5, dtype=torch.bool),
cond_len=3,
target_len=2,
seed=42,
)
captured_sampling = []
def generate_tokens(**kwargs):
captured_sampling.append((kwargs["num_step"], kwargs["guidance_scale"]))
return torch.ones(1, 8, prepared.target_len, dtype=torch.long)
pipeline = SimpleNamespace(
_prepare_request_input=lambda prompt, extra: prepared,
_collate_request_inputs=lambda requests: (prepared.input_ids, prepared.audio_mask, [prepared.cond_len]),
generator=generate_tokens,
decoder=lambda tokens: tokens.float(),
num_step=32,
guidance_scale=2.0,
t_shift=1.0,
layer_penalty_factor=0.0,
position_temperature=0.0,
class_temperature=0.0,
)
batch = DiffusionRequestBatch(
requests=[
OmniDiffusionRequest(
prompt="hello",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=12, guidance_scale=6.5),
request_id="request",
)
]
)
outputs = OmniVoicePipeline.forward(pipeline, batch)
assert outputs[0].error is None
assert captured_sampling == [(12, 6.5)]
def test_request_batch_prepare_error_preserves_surrounding_output_indices() -> None:
"""A middle prepare error must not shift valid outputs into the wrong slots."""
prepared_before = _PreparedOmniVoiceRequest(
input_ids=torch.zeros(3, 8, dtype=torch.long),
audio_mask=torch.ones(3, dtype=torch.bool),
cond_len=2,
target_len=1,
seed=1,
)
prepared_after = _PreparedOmniVoiceRequest(
input_ids=torch.zeros(5, 8, dtype=torch.long),
audio_mask=torch.ones(5, dtype=torch.bool),
cond_len=3,
target_len=2,
seed=2,
)
def prepare_request(prompt, extra):
del extra
if prompt == "before":
return prepared_before
if prompt == "after":
return prepared_after
return DiffusionOutput(error="invalid middle request")
def collate_requests(requests):
assert requests == [prepared_before, prepared_after]
return (
torch.cat([prepared_before.input_ids, prepared_after.input_ids]),
torch.cat([prepared_before.audio_mask, prepared_after.audio_mask]),
[prepared_before.cond_len, prepared_after.cond_len],
)
def generate_tokens(**kwargs):
assert kwargs["target_lens"] == [1, 2]
before = torch.full((1, 8, 1), 11, dtype=torch.long)
after = torch.full((1, 8, 2), 22, dtype=torch.long)
return torch.cat([before, after], dim=-1)
pipeline = SimpleNamespace(
_prepare_request_input=prepare_request,
_collate_request_inputs=collate_requests,
generator=generate_tokens,
decoder=lambda tokens: tokens.float(),
num_step=32,
guidance_scale=2.0,
t_shift=1.0,
layer_penalty_factor=0.0,
position_temperature=0.0,
class_temperature=0.0,
)
batch = DiffusionRequestBatch(
requests=[
OmniDiffusionRequest(
prompt=prompt,
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=32),
request_id=prompt,
)
for prompt in ("before", "bad", "after")
]
)
outputs = OmniVoicePipeline.forward(pipeline, batch)
assert len(outputs) == 3
torch.testing.assert_close(outputs[0].output, torch.full((1, 8, 1), 11.0))
assert outputs[1].error == "invalid middle request"
torch.testing.assert_close(outputs[2].output, torch.full((1, 8, 2), 22.0))