[Feat][OmniVoice]Support Varlen Attn, Request-Batch and Step-Execution (#6408)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
This commit is contained in:
1 parent
d7678e2bb3
commit
cdf027874c
14 files changed
+1396
-610
No files matched your search
@@ -6,19 +6,27 @@ Tests for OmniVoice CUDA Graph generator wrapper numerical equivalence.
|
||||
Verifies that _OmniVoiceCUDAGraphForward produces results equivalent to
|
||||
eager mode across three scenarios:
|
||||
- Exact-size inputs (no padding) → bit-identical
|
||||
- Padded inputs (padded to nearest bucket) → correct slicing, exact match
|
||||
for position-independent models
|
||||
- Oversized inputs (fallback to eager) → bit-identical
|
||||
- Padded inputs (padded to nearest bucket) → correct slicing and exact match
|
||||
- Oversized inputs (128-aligned lazy capture) → bit-identical
|
||||
|
||||
Uses a lightweight synthetic generator to keep warmup fast and avoid
|
||||
loading actual model weights.
|
||||
Uses a small randomly initialized OmniVoiceGenerator so every test exercises
|
||||
the production embedding, transformer, varlen-attention, and logits paths
|
||||
without loading checkpoint weights.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
|
||||
OmniVoiceGenerator,
|
||||
_OmniVoiceCUDAGraphForward,
|
||||
)
|
||||
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
@@ -29,53 +37,7 @@ pytestmark = [
|
||||
DEVICE = torch.device("cuda:0")
|
||||
NUM_CB = 8
|
||||
VOCAB = 1025
|
||||
HIDDEN = 64
|
||||
CAPTURE_SIZES = [32, 64, 128]
|
||||
HEAD_DIM = 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Synthetic generator: matches _OmniVoiceCUDAGraphForward's interface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _SyntheticGenerator:
|
||||
"""Position-independent Embedding+Linear replacing OmniVoiceGenerator's 28-layer transformer.
|
||||
|
||||
Each token position is processed independently (no attention), so padded
|
||||
runs produce bit-identical results to non-padded runs on the same positions.
|
||||
This makes the padding/slicing logic easy to verify exactly.
|
||||
"""
|
||||
|
||||
class _Cfg:
|
||||
num_audio_codebook = NUM_CB
|
||||
|
||||
def __init__(self, device: torch.device):
|
||||
self.config = self._Cfg()
|
||||
self.text_embedding = nn.Embedding(1000, HIDDEN).to(device).eval()
|
||||
self._linear = nn.Linear(HIDDEN, NUM_CB * VOCAB, bias=False).to(device).eval()
|
||||
self._rope_table: torch.Tensor | None = None
|
||||
|
||||
@property
|
||||
def model_dtype(self) -> torch.dtype:
|
||||
"""Part of the interface _OmniVoiceCUDAGraphForward expects of a generator."""
|
||||
return self.text_embedding.weight.dtype
|
||||
|
||||
def _rope_table_for(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""Part of the interface _OmniVoiceCUDAGraphForward expects of a generator."""
|
||||
return torch.zeros(seq_len, HEAD_DIM, device=device, dtype=dtype)
|
||||
|
||||
def _step_forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
audio_mask: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None,
|
||||
rope_table: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
two_b, _, S = input_ids.shape
|
||||
x = self.text_embedding(input_ids[:, 0, :].clamp(0, 999)) # [two_b, S, H]
|
||||
logits = self._linear(x) # [two_b, S, 8*1025]
|
||||
return logits.view(two_b, S, NUM_CB, VOCAB).permute(0, 2, 1, 3) # [two_b, 8, S, 1025]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -83,18 +45,37 @@ class _SyntheticGenerator:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _eager(gen: _SyntheticGenerator, ids: torch.Tensor, mask: torch.Tensor, attn: torch.Tensor) -> torch.Tensor:
|
||||
"""Run _step_forward with a zero RoPE table (synthetic gen ignores it)."""
|
||||
two_b, _, S = ids.shape
|
||||
return gen._step_forward(ids, mask, attn, gen._rope_table_for(S, ids.device, torch.float32))
|
||||
def _eager(gen: OmniVoiceGenerator, ids: torch.Tensor, mask: torch.Tensor, cu_seqs) -> torch.Tensor:
|
||||
"""Run the production varlen step outside CUDA Graph."""
|
||||
seq_len = ids.shape[0]
|
||||
rope_table = gen._rope_table_for(seq_len, ids.device, gen.model_dtype)
|
||||
return gen._step_forward(ids, mask, cu_seqs, rope_table)
|
||||
|
||||
|
||||
def _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, batch_size) -> torch.Tensor:
|
||||
"""Run eager with the exact padded shape and metadata used by Graph."""
|
||||
seq_len = ids.shape[0]
|
||||
bucket = wrapper._find_bucket(batch_size, seq_len)
|
||||
if bucket is None:
|
||||
bucket = round_up(seq_len, wrapper._LAZY_CAPTURE_ALIGNMENT)
|
||||
padded_ids, padded_mask = wrapper._pad_inputs(ids, mask, bucket)
|
||||
padded_cu_seqs = cu_seqs.clone()
|
||||
padded_cu_seqs[-1] = bucket
|
||||
return _eager(gen, padded_ids, padded_mask, padded_cu_seqs)[:, :seq_len, :]
|
||||
|
||||
|
||||
def _make_inputs(seq_len: int, device: torch.device = DEVICE):
|
||||
two_b = 2
|
||||
ids = torch.randint(0, 100, (two_b, NUM_CB, seq_len), dtype=torch.long, device=device)
|
||||
mask = torch.ones(two_b, seq_len, dtype=torch.bool, device=device)
|
||||
attn = torch.ones(two_b, 1, seq_len, seq_len, dtype=torch.bool, device=device)
|
||||
return ids, mask, attn
|
||||
ids = torch.randint(0, 100, (seq_len, NUM_CB), dtype=torch.long, device=device)
|
||||
mask = torch.ones(seq_len, dtype=torch.bool, device=device)
|
||||
cond_len = (seq_len + 1) // 2
|
||||
uncond_len = seq_len - cond_len
|
||||
|
||||
cu_seqs = torch.tensor(
|
||||
[0, cond_len, cond_len + uncond_len, cond_len + uncond_len],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
return ids, mask, cu_seqs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -105,15 +86,26 @@ def _make_inputs(seq_len: int, device: torch.device = DEVICE):
|
||||
@pytest.fixture(scope="module")
|
||||
def gen():
|
||||
torch.manual_seed(42)
|
||||
return _SyntheticGenerator(DEVICE)
|
||||
config = OmniVoiceConfig(
|
||||
audio_vocab_size=VOCAB,
|
||||
num_audio_codebook=NUM_CB,
|
||||
llm_config={
|
||||
"hidden_size": 64,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 2,
|
||||
"intermediate_size": 128,
|
||||
"vocab_size": 128,
|
||||
"max_position_embeddings": 4096,
|
||||
"head_dim": 16,
|
||||
},
|
||||
enable_cuda_graph=False,
|
||||
)
|
||||
return OmniVoiceGenerator(config, SimpleNamespace(max_num_seqs=2)).to(DEVICE).eval()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def wrapper(gen):
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
|
||||
_OmniVoiceCUDAGraphForward,
|
||||
)
|
||||
|
||||
w = _OmniVoiceCUDAGraphForward(gen, capture_sizes=CAPTURE_SIZES)
|
||||
w.warmup(DEVICE)
|
||||
return w
|
||||
@@ -127,10 +119,10 @@ def wrapper(gen):
|
||||
@pytest.mark.parametrize("seq_len", CAPTURE_SIZES)
|
||||
def test_exact_size_bit_identical(gen, wrapper, seq_len):
|
||||
"""When input exactly matches a captured bucket, output must be bit-identical to eager."""
|
||||
ids, mask, attn = _make_inputs(seq_len)
|
||||
ids, mask, cu_seqs = _make_inputs(seq_len)
|
||||
with torch.no_grad():
|
||||
eager_out = _eager(gen, ids, mask, attn)
|
||||
graph_out = wrapper(ids, mask, attn)
|
||||
eager_out = _eager(gen, ids, mask, cu_seqs)
|
||||
graph_out = wrapper(ids, mask, cu_seqs, 1)
|
||||
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
|
||||
|
||||
|
||||
@@ -140,40 +132,36 @@ def test_exact_size_bit_identical(gen, wrapper, seq_len):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len", [1, 15, 33, 60, 100])
|
||||
def test_padded_output_shape(gen, wrapper, seq_len):
|
||||
def test_padded_output_shape(wrapper, seq_len):
|
||||
"""Graph output must be sliced back to actual seq_len, not the bucket size."""
|
||||
ids, mask, attn = _make_inputs(seq_len)
|
||||
ids, mask, cu_seqs = _make_inputs(seq_len)
|
||||
with torch.no_grad():
|
||||
graph_out = wrapper(ids, mask, attn)
|
||||
assert graph_out.shape == (2, NUM_CB, seq_len, VOCAB)
|
||||
graph_out = wrapper(ids, mask, cu_seqs, 1)
|
||||
assert graph_out.shape == (NUM_CB, seq_len, VOCAB)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len", [15, 33, 60, 100])
|
||||
def test_padded_output_matches_eager(gen, wrapper, seq_len):
|
||||
"""Padded graph output must equal eager output at actual positions.
|
||||
|
||||
The synthetic model has no attention across positions, so zero-padding
|
||||
does not affect non-padded positions — exact match is expected.
|
||||
"""
|
||||
ids, mask, attn = _make_inputs(seq_len)
|
||||
"""Padded graph output must equal eager output at actual positions."""
|
||||
ids, mask, cu_seqs = _make_inputs(seq_len)
|
||||
with torch.no_grad():
|
||||
eager_out = _eager(gen, ids, mask, attn)
|
||||
graph_out = wrapper(ids, mask, attn)
|
||||
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 1)
|
||||
graph_out = wrapper(ids, mask, cu_seqs, 1)
|
||||
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Oversized inputs → fallback to eager (lazy capture), bit-identical
|
||||
# 3. Oversized inputs → aligned lazy capture, bit-identical
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len", [129, 200, 256])
|
||||
def test_fallback_eager_bit_identical(gen, wrapper, seq_len):
|
||||
"""Sequences exceeding the largest bucket fall back to lazy capture → bit-identical."""
|
||||
ids, mask, attn = _make_inputs(seq_len)
|
||||
def test_aligned_lazy_capture_bit_identical(gen, wrapper, seq_len):
|
||||
"""Sequences beyond the static plan use aligned lazy capture and remain bit-identical."""
|
||||
ids, mask, cu_seqs = _make_inputs(seq_len)
|
||||
with torch.no_grad():
|
||||
eager_out = _eager(gen, ids, mask, attn)
|
||||
graph_out = wrapper(ids, mask, attn)
|
||||
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 1)
|
||||
graph_out = wrapper(ids, mask, cu_seqs, 1)
|
||||
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
|
||||
|
||||
|
||||
@@ -184,13 +172,73 @@ def test_fallback_eager_bit_identical(gen, wrapper, seq_len):
|
||||
|
||||
def test_deterministic_across_calls(wrapper):
|
||||
"""Same input must produce identical output on repeated CUDA graph replays."""
|
||||
ids, mask, attn = _make_inputs(32)
|
||||
ids, mask, cu_seqs = _make_inputs(32)
|
||||
with torch.no_grad():
|
||||
out1 = wrapper(ids, mask, attn)
|
||||
out2 = wrapper(ids, mask, attn)
|
||||
out1 = wrapper(ids, mask, cu_seqs, 1).clone()
|
||||
out2 = wrapper(ids, mask, cu_seqs, 1).clone()
|
||||
torch.testing.assert_close(out1, out2, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_replay_uses_updated_cu_seqs(gen, wrapper):
|
||||
"""The same graph key must honor new sequence boundaries on replay."""
|
||||
ids, mask, _ = _make_inputs(48)
|
||||
cu_seqs_a = torch.tensor([0, 32, 48, 48], dtype=torch.int32, device=DEVICE)
|
||||
cu_seqs_b = torch.tensor([0, 17, 48, 48], dtype=torch.int32, device=DEVICE)
|
||||
|
||||
with torch.no_grad():
|
||||
eager_a = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs_a, 1)
|
||||
graph_a = wrapper(ids, mask, cu_seqs_a, 1).clone()
|
||||
eager_b = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs_b, 1)
|
||||
graph_b = wrapper(ids, mask, cu_seqs_b, 1).clone()
|
||||
|
||||
torch.testing.assert_close(graph_a, eager_a, atol=0, rtol=0)
|
||||
torch.testing.assert_close(graph_b, eager_b, atol=0, rtol=0)
|
||||
assert not torch.equal(graph_a, graph_b)
|
||||
|
||||
|
||||
def test_batch_two_cu_seqs_matches_eager(gen, wrapper):
|
||||
"""B=2 uses four real sequence segments plus the fixed tail slot."""
|
||||
ids, mask, _ = _make_inputs(48)
|
||||
cu_seqs = torch.tensor([0, 10, 18, 32, 48, 48], dtype=torch.int32, device=DEVICE)
|
||||
|
||||
assert cu_seqs.numel() == 2 * 2 + 2
|
||||
with torch.no_grad():
|
||||
eager_out = _bucketed_eager(gen, wrapper, ids, mask, cu_seqs, 2)
|
||||
graph_out = wrapper(ids, mask, cu_seqs, 2).clone()
|
||||
|
||||
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_lazy_graph_cache_uses_lru_eviction(wrapper):
|
||||
"""A lazy-cache hit must protect that graph from the next eviction."""
|
||||
original_limit = wrapper._MAX_LAZY_GRAPHS
|
||||
wrapper._lazy_graphs.clear()
|
||||
wrapper._MAX_LAZY_GRAPHS = 2
|
||||
try:
|
||||
# Static B=1 coverage ends at 128. These lengths round to three
|
||||
# distinct 128-aligned lazy keys: 256, 384, and 512.
|
||||
for seq_len in (129, 257):
|
||||
ids, mask, cu_seqs = _make_inputs(seq_len)
|
||||
with torch.no_grad():
|
||||
wrapper(ids, mask, cu_seqs, 1)
|
||||
assert list(wrapper._lazy_graphs) == [(1, 256), (1, 384)]
|
||||
|
||||
# Refresh key 256, making key 384 the least recently used entry.
|
||||
ids, mask, cu_seqs = _make_inputs(129)
|
||||
with torch.no_grad():
|
||||
wrapper(ids, mask, cu_seqs, 1)
|
||||
assert list(wrapper._lazy_graphs) == [(1, 384), (1, 256)]
|
||||
|
||||
# Inserting key 512 must evict key 384, not the recently hit key 256.
|
||||
ids, mask, cu_seqs = _make_inputs(385)
|
||||
with torch.no_grad():
|
||||
wrapper(ids, mask, cu_seqs, 1)
|
||||
assert list(wrapper._lazy_graphs) == [(1, 256), (1, 512)]
|
||||
finally:
|
||||
wrapper._MAX_LAZY_GRAPHS = original_limit
|
||||
wrapper._lazy_graphs.clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. _find_bucket logic (CPU, no CUDA graph)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -203,58 +251,17 @@ def test_find_bucket_returns_nearest_bucket():
|
||||
)
|
||||
|
||||
w = _OmniVoiceCUDAGraphForward.__new__(_OmniVoiceCUDAGraphForward)
|
||||
w._capture_sizes = [32, 64, 128]
|
||||
w.capture_bucket_sizes_by_batch = {
|
||||
1: [32, 64, 128],
|
||||
2: [64, 128, 256],
|
||||
}
|
||||
w._graphs = {}
|
||||
|
||||
assert w._find_bucket(1) == 32
|
||||
assert w._find_bucket(32) == 32
|
||||
assert w._find_bucket(33) == 64
|
||||
assert w._find_bucket(64) == 64
|
||||
assert w._find_bucket(128) == 128
|
||||
assert w._find_bucket(129) is None
|
||||
assert w._find_bucket(1000) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. enable_cuda_graph=False produces same tokens as enable_cuda_graph=True
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cuda_graph_disabled_matches_eager_generator():
|
||||
"""OmniVoiceGenerator with enable_cuda_graph=False must produce the same
|
||||
_step_forward output as one with enable_cuda_graph=True (after warmup).
|
||||
|
||||
Uses a 2-layer config so warmup completes quickly without loading weights.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import OmniVoiceGenerator
|
||||
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
|
||||
|
||||
cfg = OmniVoiceConfig()
|
||||
cfg.llm_num_hidden_layers = 2
|
||||
cfg.num_hidden_layers = 2
|
||||
|
||||
torch.manual_seed(0)
|
||||
gen_eager = OmniVoiceGenerator(cfg).to(DEVICE).eval()
|
||||
gen_eager.config.enable_cuda_graph = False
|
||||
gen_eager._cuda_graph_fwd = None
|
||||
|
||||
torch.manual_seed(0)
|
||||
gen_graph = OmniVoiceGenerator(cfg).to(DEVICE).eval()
|
||||
gen_graph.config.cuda_graph_capture_sizes = [64]
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import _OmniVoiceCUDAGraphForward
|
||||
|
||||
gen_graph._cuda_graph_fwd = _OmniVoiceCUDAGraphForward(gen_graph, capture_sizes=[64])
|
||||
gen_graph._cuda_graph_fwd.warmup(DEVICE)
|
||||
|
||||
seq_len = 64
|
||||
ids = torch.zeros(2, cfg.num_audio_codebook, seq_len, dtype=torch.long, device=DEVICE)
|
||||
mask = torch.ones(2, seq_len, dtype=torch.bool, device=DEVICE)
|
||||
attn = torch.ones(2, 1, seq_len, seq_len, dtype=torch.bool, device=DEVICE)
|
||||
|
||||
rope_table = gen_eager._rope_table_for(seq_len, DEVICE, gen_eager.model_dtype)
|
||||
|
||||
with torch.no_grad():
|
||||
eager_logits = gen_eager._step_forward(ids, mask, attn, rope_table)
|
||||
graph_logits = gen_graph._cuda_graph_fwd(ids, mask, attn)
|
||||
|
||||
torch.testing.assert_close(graph_logits, eager_logits, atol=0, rtol=0)
|
||||
assert w._find_bucket(1, 1) == 32
|
||||
assert w._find_bucket(1, 32) == 32
|
||||
assert w._find_bucket(1, 33) == 64
|
||||
assert w._find_bucket(1, 128) == 128
|
||||
assert w._find_bucket(1, 129) is None
|
||||
assert w._find_bucket(2, 33) == 64
|
||||
assert w._find_bucket(2, 129) == 256
|
||||
assert w._find_bucket(3, 64) is None
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The OmniVoice generator packs q/k/v and gate/up into fused projections.
|
||||
|
||||
The HF checkpoint stores those five tensors separately, so packing them is a
|
||||
@@ -14,6 +14,8 @@ that would leave one partly written raises instead.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -49,6 +51,13 @@ def _config() -> OmniVoiceConfig:
|
||||
)
|
||||
|
||||
|
||||
def _generator() -> OmniVoiceGenerator:
|
||||
return OmniVoiceGenerator(
|
||||
_config(),
|
||||
SimpleNamespace(max_num_seqs=1),
|
||||
)
|
||||
|
||||
|
||||
def _checkpoint_shards() -> dict[str, torch.Tensor]:
|
||||
"""The five per-layer tensors an OmniVoice checkpoint actually stores."""
|
||||
torch.manual_seed(0)
|
||||
@@ -64,7 +73,7 @@ def _checkpoint_shards() -> dict[str, torch.Tensor]:
|
||||
|
||||
|
||||
def test_every_fused_parameter_is_written() -> None:
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = _checkpoint_shards()
|
||||
|
||||
loaded = generator._load_fused_projections(shards)
|
||||
@@ -77,7 +86,7 @@ def test_every_fused_parameter_is_written() -> None:
|
||||
|
||||
def test_packed_weight_matches_the_shards_it_came_from() -> None:
|
||||
"""Packing order is q,k,v and gate,up -- the split in forward() assumes it."""
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = _checkpoint_shards()
|
||||
generator._load_fused_projections(shards)
|
||||
|
||||
@@ -101,7 +110,7 @@ def test_packed_weight_matches_the_shards_it_came_from() -> None:
|
||||
|
||||
def test_no_fused_parameter_keeps_its_random_init() -> None:
|
||||
"""The corruption this guards against: a fused param never written at all."""
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
before = {
|
||||
name: param.detach().clone()
|
||||
for name, param in generator.named_parameters()
|
||||
@@ -121,7 +130,7 @@ def test_no_fused_parameter_keeps_its_random_init() -> None:
|
||||
["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj", "mlp.gate_proj", "mlp.up_proj"],
|
||||
)
|
||||
def test_a_missing_shard_raises_instead_of_loading_partially(dropped: str) -> None:
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = _checkpoint_shards()
|
||||
del shards[f"llm.layers.0.{dropped}.weight"]
|
||||
|
||||
@@ -130,7 +139,7 @@ def test_a_missing_shard_raises_instead_of_loading_partially(dropped: str) -> No
|
||||
|
||||
|
||||
def test_a_wrong_shaped_shard_raises() -> None:
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = _checkpoint_shards()
|
||||
shards["llm.layers.0.self_attn.q_proj.weight"] = torch.randn(NUM_HEADS * HEAD_DIM + 1, HIDDEN)
|
||||
|
||||
@@ -140,7 +149,7 @@ def test_a_wrong_shaped_shard_raises() -> None:
|
||||
|
||||
def test_a_checkpoint_missing_a_whole_layer_raises() -> None:
|
||||
"""Half-loaded is the dangerous state: it neither errors nor works."""
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = {k: v for k, v in _checkpoint_shards().items() if not k.startswith("llm.layers.1.")}
|
||||
|
||||
with pytest.raises(ValueError, match="fused projections"):
|
||||
@@ -149,7 +158,7 @@ def test_a_checkpoint_missing_a_whole_layer_raises() -> None:
|
||||
|
||||
def test_a_checkpoint_with_no_fused_shards_is_left_alone() -> None:
|
||||
"""An unrelated state_dict must not trip the completeness check."""
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
assert generator._load_fused_projections({"audio_heads.weight": torch.randn(4, 4)}) == set()
|
||||
|
||||
|
||||
@@ -162,7 +171,7 @@ def test_load_weights_wires_the_packing_in(tmp_path, monkeypatch: pytest.MonkeyP
|
||||
"""
|
||||
import safetensors.torch
|
||||
|
||||
generator = OmniVoiceGenerator(_config())
|
||||
generator = _generator()
|
||||
shards = _checkpoint_shards()
|
||||
before = generator.layers[0].self_attn.qkv_proj.weight.detach().clone()
|
||||
|
||||
|
||||
@@ -1,183 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OmniVoice attention masks must follow the model dtype, not a hardcoded float32.
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""OmniVoice SDPA fallback masks must not depend on the model dtype.
|
||||
|
||||
SDPA requires the additive attention mask to carry the same dtype as the query.
|
||||
The generator used to materialize that mask as float32 unconditionally, in three
|
||||
places (the shared per-forward mask, the CUDA-graph capture buffer, and the
|
||||
replay-time normalization), so every half-precision deployment died on the first
|
||||
attention call with::
|
||||
The previous padded attention path used an additive floating-point mask. SDPA
|
||||
requires such a mask to have the same dtype as the query, and constructing it
|
||||
as float32 caused half-precision OmniVoice inference to fail on CUDA.
|
||||
|
||||
RuntimeError: invalid dtype for bias - should match query's dtype
|
||||
|
||||
(the exact wording depends on which SDPA backend is selected for the shape; the
|
||||
math backend words the same rejection as ``attn_mask.dtype``.)
|
||||
|
||||
That left OmniVoice servable only in float32, even though the upstream k2-fsa
|
||||
implementation runs it in float16.
|
||||
|
||||
Note the split between the CPU and CUDA tests below: the CPU SDPA backend
|
||||
silently accepts the mismatched mask, so only the CUDA tests can pin the actual
|
||||
crash. The CPU tests cover the mask contract itself, which is what the fix
|
||||
changes and what a future regression would break first.
|
||||
The packed-varlen path no longer needs an additive mask. When SDPA fallback is
|
||||
required, ``_attention_metadata_from_cu_seqs`` builds a boolean block mask.
|
||||
Boolean SDPA masks have no dtype coupling with the query, so no mask coercion
|
||||
is required for float16 or bfloat16 inference.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
|
||||
OmniVoiceAttention,
|
||||
OmniVoiceGenerator,
|
||||
_additive_float_mask,
|
||||
_attention_metadata_from_cu_seqs,
|
||||
)
|
||||
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
|
||||
|
||||
HALF_DTYPES = [torch.float16, torch.bfloat16]
|
||||
ALL_DTYPES = HALF_DTYPES + [torch.float32]
|
||||
|
||||
cpu_test = pytest.mark.core_model
|
||||
cuda_test = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
|
||||
def _tiny_config() -> OmniVoiceConfig:
|
||||
"""A real OmniVoiceConfig sized so the module fits in a unit test."""
|
||||
return OmniVoiceConfig(
|
||||
llm_config={
|
||||
"hidden_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 2,
|
||||
"head_dim": 8,
|
||||
"intermediate_size": 64,
|
||||
"vocab_size": 64,
|
||||
"max_position_embeddings": 128,
|
||||
},
|
||||
enable_cuda_graph=False,
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
def test_sdpa_fallback_mask_is_dtype_independent() -> None:
|
||||
"""The fallback mask is boolean rather than an additive model-dtype mask."""
|
||||
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
|
||||
|
||||
metadata = _attention_metadata_from_cu_seqs(
|
||||
cu_seqs,
|
||||
6,
|
||||
needs_sdpa_mask=True,
|
||||
)
|
||||
|
||||
|
||||
def _inputs(dtype: torch.dtype, device: torch.device) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden = torch.randn(2, 6, 32, device=device, dtype=dtype)
|
||||
bool_mask = torch.ones(2, 1, 6, 6, dtype=torch.bool, device=device)
|
||||
bool_mask[:, :, :, 4:] = False
|
||||
rope_table = torch.zeros(6, 8, device=device, dtype=dtype)
|
||||
rope_table[:, :4] = 1.0 # cos = 1, sin = 0: no rotation, so the test is about the mask
|
||||
return hidden, bool_mask, rope_table
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The mask contract (CPU)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("dtype", ALL_DTYPES)
|
||||
def test_additive_float_mask_uses_requested_dtype(dtype: torch.dtype) -> None:
|
||||
out = _additive_float_mask(torch.tensor([[True, False]]), dtype)
|
||||
assert out.dtype == dtype
|
||||
|
||||
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
def test_additive_float_mask_keeps_mask_semantics() -> None:
|
||||
"""Attend positions stay at 0.0 and masked ones at -inf, in half precision too."""
|
||||
out = _additive_float_mask(torch.tensor([[True, False]]), torch.float16)
|
||||
assert out[0, 0].item() == 0.0
|
||||
assert out[0, 1].item() == float("-inf")
|
||||
|
||||
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
def test_additive_float_mask_requires_an_explicit_dtype() -> None:
|
||||
"""The float32 default is the bug; keeping the parameter required is the fix."""
|
||||
with pytest.raises(TypeError):
|
||||
_additive_float_mask(torch.tensor([[True]])) # type: ignore[call-arg]
|
||||
|
||||
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("dtype", ALL_DTYPES)
|
||||
def test_generator_model_dtype_tracks_its_weights(dtype: torch.dtype) -> None:
|
||||
"""The single source of truth the three former float32 literals now defer to."""
|
||||
assert OmniVoiceGenerator(_tiny_config()).to(dtype).model_dtype == dtype
|
||||
|
||||
|
||||
@cpu_test
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("dtype", HALF_DTYPES)
|
||||
def test_attention_accepts_a_model_dtype_mask(dtype: torch.dtype) -> None:
|
||||
attn = OmniVoiceAttention(_tiny_config()).to(dtype).eval()
|
||||
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cpu"))
|
||||
|
||||
with torch.inference_mode():
|
||||
out = attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, dtype))
|
||||
|
||||
assert out.shape == hidden.shape
|
||||
assert out.dtype == dtype
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The crash itself (CUDA only — the CPU backend does not reject the mismatch)
|
||||
# --------------------------------------------------------------------------
|
||||
assert metadata.attn_mask is not None
|
||||
assert metadata.attn_mask.dtype == torch.bool
|
||||
|
||||
|
||||
@cuda_test
|
||||
@pytest.mark.parametrize("dtype", HALF_DTYPES)
|
||||
def test_float32_mask_is_what_broke_half_precision(dtype: torch.dtype) -> None:
|
||||
"""Pin the original failure so a float32 default cannot come back unnoticed."""
|
||||
attn = OmniVoiceAttention(_tiny_config()).to(device="cuda:0", dtype=dtype).eval()
|
||||
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cuda:0"))
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"(invalid dtype for bias|attn_mask)"), torch.inference_mode():
|
||||
attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, torch.float32))
|
||||
|
||||
|
||||
@cuda_test
|
||||
@pytest.mark.parametrize("dtype", HALF_DTYPES)
|
||||
def test_real_path_forward_runs_in_half_precision(dtype: torch.dtype) -> None:
|
||||
"""The regression, on the production path with the Triton norm kernels live."""
|
||||
attn = OmniVoiceAttention(_tiny_config()).to(device="cuda:0", dtype=dtype).eval()
|
||||
hidden, bool_mask, rope_table = _inputs(dtype, torch.device("cuda:0"))
|
||||
|
||||
with torch.inference_mode():
|
||||
out = attn(hidden, rope_table, attention_mask=_additive_float_mask(bool_mask, dtype))
|
||||
|
||||
assert out.dtype == dtype
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
@cuda_test
|
||||
@pytest.mark.parametrize("dtype", HALF_DTYPES)
|
||||
def test_generator_forward_runs_in_half_precision(dtype: torch.dtype) -> None:
|
||||
"""End-to-end over the real iterative loop, which is where the float32 mask was built.
|
||||
|
||||
This is the test that fails on an unfixed tree: ``forward`` materialized the
|
||||
shared SDPA mask as float32 regardless of the weights' dtype.
|
||||
"""
|
||||
def test_sdpa_fallback_mask_needs_no_half_precision_coercion(
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Half-precision SDPA accepts the production bool mask without casting."""
|
||||
device = torch.device("cuda:0")
|
||||
config = _tiny_config()
|
||||
generator = OmniVoiceGenerator(config).to(device=device, dtype=dtype).eval()
|
||||
seq_len = 6
|
||||
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32, device=device)
|
||||
|
||||
seq_len, target_len = 12, 4
|
||||
input_ids = torch.zeros(2, config.num_audio_codebook, seq_len, dtype=torch.long, device=device)
|
||||
input_ids[:, 1:, :] = config.audio_mask_id
|
||||
audio_mask = torch.zeros(2, seq_len, dtype=torch.bool, device=device)
|
||||
audio_mask[:, seq_len - target_len :] = True
|
||||
attention_mask = torch.ones(2, 1, seq_len, seq_len, dtype=torch.bool, device=device)
|
||||
metadata = _attention_metadata_from_cu_seqs(
|
||||
cu_seqs,
|
||||
seq_len,
|
||||
needs_sdpa_mask=True,
|
||||
)
|
||||
|
||||
assert metadata.attn_mask is not None
|
||||
assert metadata.attn_mask.dtype == torch.bool
|
||||
|
||||
query = torch.randn(1, 2, seq_len, 8, device=device, dtype=dtype)
|
||||
key = torch.randn_like(query)
|
||||
value = torch.randn_like(query)
|
||||
|
||||
with torch.inference_mode():
|
||||
tokens = generator(
|
||||
input_ids,
|
||||
audio_mask,
|
||||
attention_mask,
|
||||
target_lens=[target_len],
|
||||
seed=0,
|
||||
num_step=2,
|
||||
output = F.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=metadata.attn_mask,
|
||||
)
|
||||
|
||||
assert tokens.shape == (1, config.num_audio_codebook, target_len)
|
||||
assert tokens.dtype == torch.long
|
||||
assert output.dtype == dtype
|
||||
assert torch.isfinite(output).all()
|
||||
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.models.omnivoice.pipeline_omnivoice import (
|
||||
OmniVoicePipeline,
|
||||
_PreparedOmniVoiceRequest,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
|
||||
_attention_metadata_from_cu_seqs,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_sdpa_fallback_mask_preserves_packed_sequence_boundaries() -> None:
|
||||
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
|
||||
|
||||
metadata = _attention_metadata_from_cu_seqs(cu_seqs, 6, needs_sdpa_mask=True)
|
||||
|
||||
assert metadata.attn_mask is not None
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[1, 1, 0, 0, 0, 0],
|
||||
[1, 1, 0, 0, 0, 0],
|
||||
[0, 0, 1, 0, 0, 0],
|
||||
[0, 0, 0, 1, 1, 1],
|
||||
[0, 0, 0, 1, 1, 1],
|
||||
[0, 0, 0, 1, 1, 1],
|
||||
],
|
||||
dtype=torch.bool,
|
||||
)
|
||||
torch.testing.assert_close(metadata.attn_mask[0, 0], expected)
|
||||
|
||||
|
||||
def test_eager_attention_metadata_honors_exact_max_seqlen() -> None:
|
||||
cu_seqs = torch.tensor([0, 2, 3, 6, 6], dtype=torch.int32)
|
||||
|
||||
metadata = _attention_metadata_from_cu_seqs(
|
||||
cu_seqs,
|
||||
6,
|
||||
needs_sdpa_mask=False,
|
||||
max_seqlen=3,
|
||||
)
|
||||
|
||||
assert metadata.extra["max_seqlen_q"] == 3
|
||||
assert metadata.extra["max_seqlen_k"] == 3
|
||||
|
||||
|
||||
def test_request_batch_prepare_error_does_not_skip_valid_request() -> None:
|
||||
"""A malformed request must not prevent another request from completing."""
|
||||
prepared = _PreparedOmniVoiceRequest(
|
||||
input_ids=torch.zeros(5, 8, dtype=torch.long),
|
||||
audio_mask=torch.ones(5, dtype=torch.bool),
|
||||
cond_len=3,
|
||||
target_len=2,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
def prepare_request(prompt, extra):
|
||||
del extra
|
||||
if prompt == "bad":
|
||||
return DiffusionOutput(error="invalid prompt")
|
||||
return prepared
|
||||
|
||||
def collate_requests(requests):
|
||||
assert requests == [prepared]
|
||||
return prepared.input_ids, prepared.audio_mask, [prepared.cond_len]
|
||||
|
||||
generator_calls = []
|
||||
|
||||
def generate_tokens(**kwargs):
|
||||
generator_calls.append(kwargs)
|
||||
assert kwargs["target_lens"] == [prepared.target_len]
|
||||
return torch.ones(1, 8, prepared.target_len, dtype=torch.long)
|
||||
|
||||
pipeline = SimpleNamespace(
|
||||
_prepare_request_input=prepare_request,
|
||||
_collate_request_inputs=collate_requests,
|
||||
generator=generate_tokens,
|
||||
decoder=lambda tokens: tokens.float(),
|
||||
num_step=32,
|
||||
guidance_scale=7.0,
|
||||
t_shift=1.0,
|
||||
layer_penalty_factor=0.0,
|
||||
position_temperature=0.0,
|
||||
class_temperature=0.0,
|
||||
)
|
||||
sampling = OmniDiffusionSamplingParams(num_inference_steps=32, guidance_scale=7.0)
|
||||
batch = DiffusionRequestBatch(
|
||||
requests=[
|
||||
OmniDiffusionRequest(prompt="bad", sampling_params=sampling, request_id="bad"),
|
||||
OmniDiffusionRequest(
|
||||
prompt="good",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=32, guidance_scale=7.0),
|
||||
request_id="good",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
outputs = OmniVoicePipeline.forward(pipeline, batch)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert outputs[0].error == "invalid prompt"
|
||||
assert outputs[1].error is None
|
||||
torch.testing.assert_close(outputs[1].output, torch.ones(1, 8, 2))
|
||||
assert generator_calls[0]["guidance_scale"] == 7.0
|
||||
|
||||
|
||||
def test_request_batch_honors_explicit_sampling_overrides() -> None:
|
||||
"""Request steps and guidance must override the OmniVoice defaults."""
|
||||
prepared = _PreparedOmniVoiceRequest(
|
||||
input_ids=torch.zeros(5, 8, dtype=torch.long),
|
||||
audio_mask=torch.ones(5, dtype=torch.bool),
|
||||
cond_len=3,
|
||||
target_len=2,
|
||||
seed=42,
|
||||
)
|
||||
captured_sampling = []
|
||||
|
||||
def generate_tokens(**kwargs):
|
||||
captured_sampling.append((kwargs["num_step"], kwargs["guidance_scale"]))
|
||||
return torch.ones(1, 8, prepared.target_len, dtype=torch.long)
|
||||
|
||||
pipeline = SimpleNamespace(
|
||||
_prepare_request_input=lambda prompt, extra: prepared,
|
||||
_collate_request_inputs=lambda requests: (prepared.input_ids, prepared.audio_mask, [prepared.cond_len]),
|
||||
generator=generate_tokens,
|
||||
decoder=lambda tokens: tokens.float(),
|
||||
num_step=32,
|
||||
guidance_scale=2.0,
|
||||
t_shift=1.0,
|
||||
layer_penalty_factor=0.0,
|
||||
position_temperature=0.0,
|
||||
class_temperature=0.0,
|
||||
)
|
||||
batch = DiffusionRequestBatch(
|
||||
requests=[
|
||||
OmniDiffusionRequest(
|
||||
prompt="hello",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=12, guidance_scale=6.5),
|
||||
request_id="request",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
outputs = OmniVoicePipeline.forward(pipeline, batch)
|
||||
|
||||
assert outputs[0].error is None
|
||||
assert captured_sampling == [(12, 6.5)]
|
||||
|
||||
|
||||
def test_request_batch_prepare_error_preserves_surrounding_output_indices() -> None:
|
||||
"""A middle prepare error must not shift valid outputs into the wrong slots."""
|
||||
prepared_before = _PreparedOmniVoiceRequest(
|
||||
input_ids=torch.zeros(3, 8, dtype=torch.long),
|
||||
audio_mask=torch.ones(3, dtype=torch.bool),
|
||||
cond_len=2,
|
||||
target_len=1,
|
||||
seed=1,
|
||||
)
|
||||
prepared_after = _PreparedOmniVoiceRequest(
|
||||
input_ids=torch.zeros(5, 8, dtype=torch.long),
|
||||
audio_mask=torch.ones(5, dtype=torch.bool),
|
||||
cond_len=3,
|
||||
target_len=2,
|
||||
seed=2,
|
||||
)
|
||||
|
||||
def prepare_request(prompt, extra):
|
||||
del extra
|
||||
if prompt == "before":
|
||||
return prepared_before
|
||||
if prompt == "after":
|
||||
return prepared_after
|
||||
return DiffusionOutput(error="invalid middle request")
|
||||
|
||||
def collate_requests(requests):
|
||||
assert requests == [prepared_before, prepared_after]
|
||||
return (
|
||||
torch.cat([prepared_before.input_ids, prepared_after.input_ids]),
|
||||
torch.cat([prepared_before.audio_mask, prepared_after.audio_mask]),
|
||||
[prepared_before.cond_len, prepared_after.cond_len],
|
||||
)
|
||||
|
||||
def generate_tokens(**kwargs):
|
||||
assert kwargs["target_lens"] == [1, 2]
|
||||
before = torch.full((1, 8, 1), 11, dtype=torch.long)
|
||||
after = torch.full((1, 8, 2), 22, dtype=torch.long)
|
||||
return torch.cat([before, after], dim=-1)
|
||||
|
||||
pipeline = SimpleNamespace(
|
||||
_prepare_request_input=prepare_request,
|
||||
_collate_request_inputs=collate_requests,
|
||||
generator=generate_tokens,
|
||||
decoder=lambda tokens: tokens.float(),
|
||||
num_step=32,
|
||||
guidance_scale=2.0,
|
||||
t_shift=1.0,
|
||||
layer_penalty_factor=0.0,
|
||||
position_temperature=0.0,
|
||||
class_temperature=0.0,
|
||||
)
|
||||
batch = DiffusionRequestBatch(
|
||||
requests=[
|
||||
OmniDiffusionRequest(
|
||||
prompt=prompt,
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=32),
|
||||
request_id=prompt,
|
||||
)
|
||||
for prompt in ("before", "bad", "after")
|
||||
]
|
||||
)
|
||||
|
||||
outputs = OmniVoicePipeline.forward(pipeline, batch)
|
||||
|
||||
assert len(outputs) == 3
|
||||
torch.testing.assert_close(outputs[0].output, torch.full((1, 8, 1), 11.0))
|
||||
assert outputs[1].error == "invalid middle request"
|
||||
torch.testing.assert_close(outputs[2].output, torch.full((1, 8, 2), 22.0))
|
||||
Reference in new issue
Block a user