[Benchmark] Universal TTS benchmark: Qwen3-TTS + VoxCPM2 with 3 task types (voice-clone/default/design) (#2835)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-04-24 23:35:33 +08:00
1 parent d388583e0b
commit 39e3fd3d12
29 files changed
+1840 -2799

No files matched your search

+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""conftest.py for benchmarks unit tests.
Installs lightweight mock stubs for ``vllm`` (and sub-packages) so the
data-module unit tests can run without a full vLLM installation. Only the
symbols actually imported by
``vllm_omni.benchmarks.data_modules.seed_tts_dataset`` are emulated.
"""
from __future__ import annotations
import sys
import types
from dataclasses import dataclass
from typing import Any
def _install_vllm_stubs() -> None:
"""Register minimal vllm stubs in sys.modules.
Only installs when real vllm is unavailable. We actively probe the
import because an empty or partial vllm may not yet have imported
the submodules we rely on, and unconditionally registering stubs
would shadow the real package for sibling tests (e.g.
``tests/benchmarks/metrics/test_metrics.py`` needs the real
``vllm.benchmarks.serve``).
"""
try:
import vllm.benchmarks.datasets # noqa: F401
import vllm.tokenizers # noqa: F401
except ImportError:
pass
else:
return # real vllm available — do not shadow it
if "vllm.benchmarks.datasets" in sys.modules:
return
# ------------------------------------------------------------------ #
# vllm.benchmarks.datasets #
# ------------------------------------------------------------------ #
@dataclass
class SampleRequest:
prompt: str = ""
prompt_len: int = 0
expected_output_len: int = 0
multi_modal_data: Any = None
request_id: str = ""
class BenchmarkDataset:
def __init__(
self,
dataset_path: str = "",
random_seed: int = 0,
disable_shuffle: bool = False,
**kwargs: Any,
) -> None:
self.dataset_path = dataset_path
self.random_seed = random_seed
self.disable_shuffle = disable_shuffle
def maybe_oversample_requests(
self,
out: list,
num_requests: int,
request_id_prefix: str,
no_oversample: bool,
) -> None:
pass
# ------------------------------------------------------------------ #
# vllm.tokenizers / vllm.tokenizers.hf #
# ------------------------------------------------------------------ #
class TokenizerLike:
pass
def get_cached_tokenizer(t: Any) -> Any:
return t
# ------------------------------------------------------------------ #
# Wire up sys.modules #
# ------------------------------------------------------------------ #
vllm_mod = types.ModuleType("vllm")
vllm_benchmarks = types.ModuleType("vllm.benchmarks")
vllm_benchmarks_datasets = types.ModuleType("vllm.benchmarks.datasets")
vllm_tokenizers = types.ModuleType("vllm.tokenizers")
vllm_tokenizers_hf = types.ModuleType("vllm.tokenizers.hf")
vllm_benchmarks_datasets.BenchmarkDataset = BenchmarkDataset # type: ignore[attr-defined]
vllm_benchmarks_datasets.SampleRequest = SampleRequest # type: ignore[attr-defined]
vllm_tokenizers.TokenizerLike = TokenizerLike # type: ignore[attr-defined]
vllm_tokenizers_hf.get_cached_tokenizer = get_cached_tokenizer # type: ignore[attr-defined]
sys.modules["vllm"] = vllm_mod
sys.modules["vllm.benchmarks"] = vllm_benchmarks
sys.modules["vllm.benchmarks.datasets"] = vllm_benchmarks_datasets
sys.modules["vllm.tokenizers"] = vllm_tokenizers
sys.modules["vllm.tokenizers.hf"] = vllm_tokenizers_hf
# Install stubs immediately at collection time (before any test import).
_install_vllm_stubs()
+139
View File
@@ -0,0 +1,139 @@
"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
import yaml
# Add benchmarks/tts to path for import
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "benchmarks" / "tts"))
import bench_tts
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture()
def model_configs_path(tmp_path: Path) -> Path:
cfg = {
"models": {
"test/ModelA": {
"stage_config": "model_a.yaml",
"supported_tasks": ["voice_clone", "default_voice"],
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"task_extra_body": {
"voice_clone": {"task_type": "Base"},
"default_voice": {"voice": "Vivian", "task_type": "CustomVoice"},
},
},
"test/ModelB": {
"stage_config": "model_b.yaml",
"supported_tasks": ["voice_clone"],
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"task_extra_body": {"voice_clone": {}},
},
}
}
p = tmp_path / "model_configs.yaml"
p.write_text(yaml.dump(cfg), encoding="utf-8")
return p
def test_load_model_configs(model_configs_path: Path) -> None:
configs = bench_tts.load_model_configs(model_configs_path)
assert "test/ModelA" in configs
assert "test/ModelB" in configs
assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
def test_build_bench_args_voice_clone(model_configs_path: Path) -> None:
configs = bench_tts.load_model_configs(model_configs_path)
cmd = bench_tts.build_bench_args(
host="localhost",
port=8000,
model="test/ModelA",
task="voice_clone",
model_cfg=configs["test/ModelA"],
locale="en",
num_prompts=10,
concurrency=1,
dataset_path="/data/seed-tts",
wer_eval=False,
output_dir=None,
result_filename=None,
extra_cli_args=[],
)
assert "--dataset-name" in cmd
idx = cmd.index("--dataset-name")
assert cmd[idx + 1] == "seed-tts"
assert "--max-concurrency" in cmd
assert "--extra-body" in cmd
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
assert extra_body.get("task_type") == "Base"
def test_build_bench_args_default_voice_has_voice_param(model_configs_path: Path) -> None:
configs = bench_tts.load_model_configs(model_configs_path)
cmd = bench_tts.build_bench_args(
host="localhost",
port=8000,
model="test/ModelA",
task="default_voice",
model_cfg=configs["test/ModelA"],
locale="en",
num_prompts=10,
concurrency=1,
dataset_path="/data/seed-tts",
wer_eval=False,
output_dir=None,
result_filename=None,
extra_cli_args=[],
)
idx = cmd.index("--dataset-name")
assert cmd[idx + 1] == "seed-tts-text"
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
assert extra_body.get("voice") == "Vivian"
def test_build_bench_args_wer_eval_adds_flag(model_configs_path: Path) -> None:
configs = bench_tts.load_model_configs(model_configs_path)
cmd = bench_tts.build_bench_args(
host="localhost",
port=8000,
model="test/ModelA",
task="voice_clone",
model_cfg=configs["test/ModelA"],
locale="en",
num_prompts=10,
concurrency=1,
dataset_path="/data/seed-tts",
wer_eval=True,
output_dir=None,
result_filename=None,
extra_cli_args=[],
)
assert "--seed-tts-wer-eval" in cmd
def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.CaptureFixture, mocker) -> None:
# ModelB does not support voice_design
mocker.patch.object(
sys,
"argv",
[
"bench_tts.py",
"--model",
"test/ModelB",
"--task",
"voice_design",
"--model-configs",
str(model_configs_path),
],
)
with pytest.raises(SystemExit):
bench_tts.main()
@@ -0,0 +1,159 @@
"""Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
and SeedTTSDesignSampleRequest.
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
"""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
_REPO_ROOT = Path(__file__).resolve().parents[2]
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "seed_tts_dataset.py"
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"
if _MODULE_NAME not in sys.modules:
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
_mod = importlib.util.module_from_spec(_spec)
sys.modules[_MODULE_NAME] = _mod
_spec.loader.exec_module(_mod)
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
SeedTTSDesignDataset,
SeedTTSDesignSampleRequest,
SeedTTSTextDataset,
SeedTTSTextSampleRequest,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture()
def seed_tts_root(tmp_path: Path) -> Path:
"""Minimal seed-tts-style directory with 5 entries."""
locale_dir = tmp_path / "en"
locale_dir.mkdir()
wav_dir = locale_dir / "prompt-wavs"
wav_dir.mkdir()
for i in range(5):
(wav_dir / f"utt{i:03d}.wav").write_bytes(b"RIFF\x00\x00\x00\x00WAVE")
meta = "\n".join(f"utt{i:03d}|ref text {i}|prompt-wavs/utt{i:03d}.wav|target text {i}" for i in range(5))
(locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
return tmp_path
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mocker):
ds = SeedTTSTextDataset(
dataset_path=str(seed_tts_root),
random_seed=0,
locale="en",
disable_shuffle=True,
)
tokenizer = mocker.MagicMock()
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
requests = ds.sample(tokenizer, num_requests=3)
assert len(requests) == 3
for req in requests:
assert isinstance(req, SeedTTSTextSampleRequest)
assert req.seed_tts_speech_extra is None or "ref_audio" not in (req.seed_tts_speech_extra or {})
assert req.seed_tts_ref_wav_path == ""
assert "target text" in req.prompt
# ---------------------------------------------------------------------------
# SeedTTSDesignDataset tests
# ---------------------------------------------------------------------------
@pytest.fixture()
def seed_tts_design_root(tmp_path: Path) -> Path:
"""seed-tts-design directory with 5-field meta.lst entries."""
locale_dir = tmp_path / "en"
locale_dir.mkdir()
meta = "\n".join(
f"des{i:03d}|||target text {i}|A warm {['female', 'male'][i % 2]} voice with neutral accent." for i in range(5)
)
(locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
return tmp_path
def test_seed_tts_design_dataset_has_instructions(seed_tts_design_root, mocker):
ds = SeedTTSDesignDataset(
dataset_path=str(seed_tts_design_root),
random_seed=0,
locale="en",
disable_shuffle=True,
)
tokenizer = mocker.MagicMock()
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
requests = ds.sample(tokenizer, num_requests=3)
assert len(requests) == 3
for req in requests:
assert isinstance(req, SeedTTSDesignSampleRequest)
extra = req.seed_tts_speech_extra or {}
assert "instructions" in extra
assert extra["instructions"], "instructions must be non-empty"
assert extra.get("task_type") == "VoiceDesign"
assert "ref_audio" not in extra
assert req.seed_tts_ref_wav_path == ""
def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_root, mocker):
"""Lines without a voice_description should be skipped."""
locale_dir = seed_tts_design_root / "en"
(locale_dir / "meta.lst").write_text(
"bad001|||target text without description\nok001|||good target|A clear female voice.\n",
encoding="utf-8",
)
ds = SeedTTSDesignDataset(
dataset_path=str(seed_tts_design_root),
random_seed=0,
locale="en",
disable_shuffle=True,
)
tokenizer = mocker.MagicMock()
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
requests = ds.sample(tokenizer, num_requests=10)
assert len(requests) == 1 # only the valid row
def test_attach_sets_seed_tts_row_even_without_extra_body():
"""seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
req = SeedTTSTextSampleRequest(
prompt="hello world",
prompt_len=2,
expected_output_len=100,
multi_modal_data=None,
request_id="test-0",
seed_tts_speech_extra=None,
seed_tts_ref_wav_path="",
)
assert req.seed_tts_speech_extra is None
assert req.seed_tts_ref_wav_path == ""
# The fix ensures that even with speech_extra=None, the function
# sets seed_tts_row=True. We verify the source code has the fix.
import inspect
import vllm_omni.benchmarks.patch.patch as patch_mod
src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
# seed_tts_row must be set BEFORE the 'if not ex: return' check
row_pos = src.index("seed_tts_row")
not_ex_pos = src.index("if not ex:")
assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"