[Refactor][CI] Reorganize test tree, and move slow e2e to weekly (#4951)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-07-29 23:53:03 +08:00
1 parent 925347332f
commit 7c17111ff5
158 files changed
+3479 -4130

No files matched your search

@@ -20,8 +20,9 @@ import pytest
import torch
import torch.nn as nn
pytestmark = [pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")]
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.core_model]
DEVICE = torch.device("cuda:0")
NUM_QUANTIZERS = 8
TOTAL_UPSAMPLE = 4
@@ -112,6 +113,7 @@ def _random_codes(seq_len, batch_size=1, device=DEVICE):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [25, 50, 100])
def test_exact_size_numerical_equivalence(decoder, wrapper, seq_len):
"""When input exactly matches a capture size, output must be bit-identical."""
@@ -127,6 +129,7 @@ def test_exact_size_numerical_equivalence(decoder, wrapper, seq_len):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_output_shape_and_length(decoder, wrapper, seq_len):
"""Padded decode must return output trimmed to actual input length."""
@@ -139,6 +142,7 @@ def test_padded_output_shape_and_length(decoder, wrapper, seq_len):
assert graph_out.shape[-1] == expected_len
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_interior_positions_close(decoder, wrapper, seq_len):
"""Interior positions (away from padding boundary) should be very close.
@@ -160,6 +164,7 @@ def test_padded_interior_positions_close(decoder, wrapper, seq_len):
torch.testing.assert_close(interior_graph, interior_eager, atol=1e-5, rtol=1e-5)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_output_bounded(decoder, wrapper, seq_len):
"""Padded output values must remain in [-1, 1] and max diff should be bounded."""
@@ -175,6 +180,7 @@ def test_padded_output_bounded(decoder, wrapper, seq_len):
assert max_diff < 1.0, f"Max diff {max_diff} exceeds bound"
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_short_output_decode_length_matches_eager(seq_len):
"""Streaming ``decode()`` of a shorter-than-nominal decoder must match eager length.
@@ -210,6 +216,7 @@ def test_padded_short_output_decode_length_matches_eager(seq_len):
torch.testing.assert_close(graph_out[..., :-boundary], eager_out[..., :-boundary], atol=1e-5, rtol=1e-5)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_compiled_padded_short_output_length_matches_eager(monkeypatch):
"""Compiled replay path must also trim short-output decoders to eager length.
@@ -251,6 +258,7 @@ def test_compiled_padded_short_output_length_matches_eager(monkeypatch):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("seq_len", [101, 150, 200])
def test_fallback_eager_exact_match(decoder, wrapper, seq_len):
"""Input larger than all capture sizes falls back to eager -> bit-identical."""
@@ -266,6 +274,7 @@ def test_fallback_eager_exact_match(decoder, wrapper, seq_len):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("total_len", [60, 100, 150, 250])
def test_chunked_decode_shape_match(decoder, wrapper, total_len):
"""Chunked decode output shape must match between eager and graph modes."""
@@ -279,6 +288,7 @@ def test_chunked_decode_shape_match(decoder, wrapper, total_len):
assert eager_out.shape == graph_out.shape
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("total_len", [50, 100])
def test_chunked_decode_exact_size_equivalence(decoder, wrapper, total_len):
"""Chunked decode with chunks matching capture sizes should be bit-identical."""
@@ -293,6 +303,7 @@ def test_chunked_decode_exact_size_equivalence(decoder, wrapper, total_len):
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_chunked_decode_output_survives_later_replay(wrapper):
"""Chunked output must not alias graph static buffers overwritten by later replays."""
codes = _random_codes(100)
@@ -307,6 +318,7 @@ def test_chunked_decode_output_survives_later_replay(wrapper):
torch.testing.assert_close(graph_out, expected, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_chunked_decode_preserves_short_chunk_concat_semantics():
"""Chunked decode must allow shorter-than-nominal chunk outputs.
@@ -336,6 +348,7 @@ def test_chunked_decode_preserves_short_chunk_concat_semantics():
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_batched_chunked_decode_variable_lengths_matches_per_request_eager(decoder, wrapper):
"""Variable-length chunk batching should match independent chunked decodes."""
long_codes = _random_codes(100)
@@ -380,6 +393,7 @@ def _eager_chunked(decoder, codes, chunk_size, left_context_size):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_single_frame(decoder, wrapper):
"""Single-frame input (seq_len=1) should work with padding."""
codes = _random_codes(1)
@@ -390,6 +404,7 @@ def test_single_frame(decoder, wrapper):
assert graph_out.shape[-1] == TOTAL_UPSAMPLE
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_disabled_wrapper_matches_eager(decoder, wrapper):
"""Disabled wrapper should produce bit-identical output to eager."""
codes = _random_codes(30)
@@ -401,6 +416,7 @@ def test_disabled_wrapper_matches_eager(decoder, wrapper):
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_batch_size_gt1_uses_matching_graph(decoder, wrapper):
"""Captured batch size > 1 should replay a matching graph."""
assert (2, 25) in wrapper.graphs
@@ -411,6 +427,7 @@ def test_batch_size_gt1_uses_matching_graph(decoder, wrapper):
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_uncaptured_batch_size_falls_back(decoder, wrapper):
"""Uncaptured batch sizes should fall back to eager."""
assert (3, 25) not in wrapper.graphs
@@ -421,6 +438,7 @@ def test_uncaptured_batch_size_falls_back(decoder, wrapper):
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_extra_capture_shape_uses_sparse_graph(decoder):
"""Extra capture shapes should not expand to a full batch x size product."""
sparse_wrapper = CUDAGraphDecoderWrapper(
@@ -444,6 +462,7 @@ def test_extra_capture_shape_uses_sparse_graph(decoder):
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_compile_shape_supports_exact_and_padded_buckets(decoder, monkeypatch):
"""Configured torch.compile shapes should replay exact and padded CUDA Graph buckets."""
@@ -490,6 +509,7 @@ def test_compile_shape_supports_exact_and_padded_buckets(decoder, monkeypatch):
assert compile_kwargs["dynamic"] is False
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_deterministic_across_calls(decoder, wrapper):
"""Same input should produce identical CUDA graph output across calls."""
codes = _random_codes(30)
@@ -504,6 +524,7 @@ def test_deterministic_across_calls(decoder, wrapper):
# ──────────────────────────────────────────────────────────────────
@pytest.mark.cpu
@pytest.mark.parametrize(
"kwargs,expected_in,not_expected",
[
@@ -545,6 +566,7 @@ def test_compute_capture_sizes(kwargs, expected_in, not_expected):
# ──────────────────────────────────────────────────────────────────
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize(
"batch,channels,seq_len",
[(2, 64, 1000), (1, 32, 1), (1, 32, 7), (1, 32, 128), (1, 32, 1024), (1, 32, 4096)],
@@ -17,6 +17,8 @@ from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
Qwen3TTSTalkerForConditionalGeneration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_minimal_talker(
tts_pad_embed: torch.Tensor | None = None,
@@ -0,0 +1,283 @@
import json
from types import SimpleNamespace
import pytest
import torch
from safetensors.torch import save_file
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _write_manifest(root, *, model_type: str, voices: dict) -> None:
payload = {
"schema_version": 1,
"model_type": model_type,
"voices": voices,
}
(root / "custom_voice_manifest.json").write_text(json.dumps(payload), encoding="utf-8")
def test_qwen3_custom_voice_profiles_warm_speaker_cache(tmp_path):
from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
Qwen3TTSTalkerForConditionalGeneration,
)
save_file(
{
"speaker_embedding": torch.arange(4, dtype=torch.float32),
"ref_code": torch.arange(6, dtype=torch.int32).reshape(3, 2),
},
str(tmp_path / "alice.safetensors"),
)
_write_manifest(
tmp_path,
model_type="qwen3_tts",
voices={
"Alice": {
"file": "alice.safetensors",
"mode": "icl",
"ref_text": "reference transcript",
"embedding_dim": 4,
}
},
)
model = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration)
model.config = SimpleNamespace(
custom_voice_dir=str(tmp_path),
speaker_encoder_config=SimpleNamespace(enc_dim=4),
)
model._speaker_cache = SpeakerEmbeddingCache()
model._load_custom_voice_profiles()
key = model._speaker_cache.make_cache_key("alice", model_type="qwen3_tts_icl", created_at=0)
cached = model._speaker_cache.get(key)
assert cached is not None
assert cached["icl_mode"] is True
assert cached["ref_text"] == "reference transcript"
torch.testing.assert_close(cached["ref_spk_embedding"], torch.arange(4, dtype=torch.float32))
torch.testing.assert_close(cached["ref_code"], torch.arange(6, dtype=torch.int32).reshape(3, 2))
def test_qwen3_icl_profile_without_ref_code_is_not_downgraded(tmp_path):
from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
Qwen3TTSTalkerForConditionalGeneration,
)
save_file({"speaker_embedding": torch.arange(4, dtype=torch.float32)}, str(tmp_path / "alice.safetensors"))
_write_manifest(
tmp_path,
model_type="qwen3_tts",
voices={
"Alice": {
"file": "alice.safetensors",
"mode": "icl",
"ref_text": "reference transcript",
"embedding_dim": 4,
}
},
)
model = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration)
model.config = SimpleNamespace(
custom_voice_dir=str(tmp_path),
speaker_encoder_config=SimpleNamespace(enc_dim=4),
)
model._speaker_cache = SpeakerEmbeddingCache()
model._load_custom_voice_profiles()
icl_key = model._speaker_cache.make_cache_key("alice", model_type="qwen3_tts_icl", created_at=0)
xvec_key = model._speaker_cache.make_cache_key("alice", model_type="qwen3_tts_xvec", created_at=0)
assert model._speaker_cache.get(icl_key) is None
assert model._speaker_cache.get(xvec_key) is None
def test_qwen3_prompt_len_uses_precomputed_ref_code_length():
from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import (
Qwen3TTSPromptEmbedsBuilder,
)
prompt_len = Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information(
additional_information={
"text": ["hello"],
"task_type": ["Base"],
"speaker": ["alice"],
"x_vector_only_mode": [False],
"ref_code_length": [6],
"ref_text": ["reference transcript"],
},
task_type="Base",
tokenize_prompt=lambda _text: list(range(10)),
codec_language_id=None,
spk_is_dialect=None,
estimate_ref_code_len=lambda _ref_audio: None,
)
assert prompt_len == 15
def test_custom_voice_profile_file_must_stay_under_manifest_dir(tmp_path):
from vllm_omni.utils.speaker_cache import iter_custom_voice_profiles
_write_manifest(
tmp_path,
model_type="qwen3_tts",
voices={"Alice": {"file": "../alice.safetensors", "mode": "xvec"}},
)
assert iter_custom_voice_profiles(tmp_path, expected_model_type="qwen3_tts") == []
def test_custom_voice_profiles_drop_stale_provenance_metadata(tmp_path):
from vllm_omni.utils.speaker_cache import iter_custom_voice_profiles
save_file({"speaker_embedding": torch.arange(4, dtype=torch.float32)}, str(tmp_path / "alice.safetensors"))
_write_manifest(
tmp_path,
model_type="qwen3_tts",
voices={
"Alice": {
"file": "alice.safetensors",
"mode": "xvec",
"audio_codec_config_hash": "old-hash",
"audio_codec_version": "old-codec",
"model_revision": "old-revision",
}
},
)
profiles = iter_custom_voice_profiles(tmp_path, expected_model_type="qwen3_tts")
assert len(profiles) == 1
assert "audio_codec_config_hash" not in profiles[0]
assert "audio_codec_version" not in profiles[0]
assert "model_revision" not in profiles[0]
def test_voxcpm2_custom_voice_profiles_warm_full_prompt_cache(tmp_path):
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
VoxCPM2TalkerForConditionalGeneration,
)
save_file(
{
"ref_audio_feat": torch.ones(2, 3, 4, dtype=torch.float32),
"audio_feat": torch.full((5, 3, 4), 2.0, dtype=torch.float32),
},
str(tmp_path / "bob.safetensors"),
)
_write_manifest(
tmp_path,
model_type="voxcpm2",
voices={
"Bob": {
"file": "bob.safetensors",
"mode": "ref_continuation",
"prompt_text": "prompt transcript",
"ref_audio_feat_len": 2,
"audio_feat_len": 5,
}
},
)
model = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
model.config = SimpleNamespace(custom_voice_dir=str(tmp_path))
model._speaker_cache = SpeakerEmbeddingCache()
model._load_custom_voice_profiles()
key = model._speaker_cache.make_cache_key("bob", model_type="voxcpm2", created_at=0)
cached = model._speaker_cache.get(key)
assert cached is not None
assert cached["mode"] == "ref_continuation"
assert cached["prompt_text"] == "prompt transcript"
torch.testing.assert_close(cached["ref_audio_feat"], torch.ones(2, 3, 4))
torch.testing.assert_close(cached["audio_feat"], torch.full((5, 3, 4), 2.0))
def test_voxcpm2_custom_voice_profiles_skip_unloadable_manifest_entry(tmp_path):
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
VoxCPM2TalkerForConditionalGeneration,
)
_write_manifest(
tmp_path,
model_type="voxcpm2",
voices={
"Bob": {
"file": "missing.safetensors",
"mode": "reference",
"ref_audio_feat_len": 2,
}
},
)
model = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
model.config = SimpleNamespace(custom_voice_dir=str(tmp_path))
model._speaker_cache = SpeakerEmbeddingCache()
model._load_custom_voice_profiles()
key = model._speaker_cache.make_cache_key("bob", model_type="voxcpm2", created_at=0)
assert model._speaker_cache.get(key) is None
def test_voxcpm2_precomputed_voice_cache_miss_raises_before_zero_shot():
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
VoxCPM2TalkerForConditionalGeneration,
)
model = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
model.config = SimpleNamespace(bos_token_id=1)
model._speaker_cache = SpeakerEmbeddingCache()
model._get_multichar_zh_split = lambda: {}
model._get_or_create_state = lambda _req_id: SimpleNamespace()
with pytest.raises(ValueError, match="not loaded in the model cache"):
model.preprocess(
torch.tensor([1, 2], dtype=torch.long),
None,
request_id="req-1",
text_token_ids=[[2]],
voice_name=["bob"],
voice_created_at=0,
voice_profile={"mode": "reference"},
)
def test_voxcpm2_prompt_can_account_for_precomputed_profile_lengths():
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import build_voxcpm2_prompt
class _Tokenizer:
bos_token_id = 1
unk_token_id = 0
def get_vocab(self):
return {}
def encode(self, text, add_special_tokens=True):
ids = [10 + i for i, _ in enumerate(text.split())]
return [self.bos_token_id, *ids] if add_special_tokens else ids
prompt = build_voxcpm2_prompt(
hf_config=SimpleNamespace(audio_vae_config={"sample_rate": 48000, "encoder_rates": [2]}, patch_size=3),
tokenizer=_Tokenizer(),
split_map={},
text="hello world",
voice_profile={
"mode": "ref_continuation",
"ref_audio_feat_len": 2,
"audio_feat_len": 5,
"prompt_text": "prompt words",
},
)
# text(2) + audio_start(1) + ref prefix(2 + ref_len) + prompt_text(2) + prompt_audio_len(5)
assert len(prompt["prompt_token_ids"]) == 14
additional = prompt["additional_information"]
assert additional["voice_profile"]["mode"] == "ref_continuation"
assert additional["prompt_text"] == ["prompt words"]