[Benchmark] Universal TTS benchmark: Qwen3-TTS + VoxCPM2 with 3 task types (voice-clone/default/design) (#2835)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
d388583e0b
commit
39e3fd3d12
29 files changed
+1840
-2799
No files matched your search
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""conftest.py for benchmarks unit tests.
|
||||
|
||||
Installs lightweight mock stubs for ``vllm`` (and sub-packages) so the
|
||||
data-module unit tests can run without a full vLLM installation. Only the
|
||||
symbols actually imported by
|
||||
``vllm_omni.benchmarks.data_modules.seed_tts_dataset`` are emulated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _install_vllm_stubs() -> None:
|
||||
"""Register minimal vllm stubs in sys.modules.
|
||||
|
||||
Only installs when real vllm is unavailable. We actively probe the
|
||||
import because an empty or partial vllm may not yet have imported
|
||||
the submodules we rely on, and unconditionally registering stubs
|
||||
would shadow the real package for sibling tests (e.g.
|
||||
``tests/benchmarks/metrics/test_metrics.py`` needs the real
|
||||
``vllm.benchmarks.serve``).
|
||||
"""
|
||||
try:
|
||||
import vllm.benchmarks.datasets # noqa: F401
|
||||
import vllm.tokenizers # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
return # real vllm available — do not shadow it
|
||||
if "vllm.benchmarks.datasets" in sys.modules:
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# vllm.benchmarks.datasets #
|
||||
# ------------------------------------------------------------------ #
|
||||
@dataclass
|
||||
class SampleRequest:
|
||||
prompt: str = ""
|
||||
prompt_len: int = 0
|
||||
expected_output_len: int = 0
|
||||
multi_modal_data: Any = None
|
||||
request_id: str = ""
|
||||
|
||||
class BenchmarkDataset:
|
||||
def __init__(
|
||||
self,
|
||||
dataset_path: str = "",
|
||||
random_seed: int = 0,
|
||||
disable_shuffle: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self.dataset_path = dataset_path
|
||||
self.random_seed = random_seed
|
||||
self.disable_shuffle = disable_shuffle
|
||||
|
||||
def maybe_oversample_requests(
|
||||
self,
|
||||
out: list,
|
||||
num_requests: int,
|
||||
request_id_prefix: str,
|
||||
no_oversample: bool,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# vllm.tokenizers / vllm.tokenizers.hf #
|
||||
# ------------------------------------------------------------------ #
|
||||
class TokenizerLike:
|
||||
pass
|
||||
|
||||
def get_cached_tokenizer(t: Any) -> Any:
|
||||
return t
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Wire up sys.modules #
|
||||
# ------------------------------------------------------------------ #
|
||||
vllm_mod = types.ModuleType("vllm")
|
||||
vllm_benchmarks = types.ModuleType("vllm.benchmarks")
|
||||
vllm_benchmarks_datasets = types.ModuleType("vllm.benchmarks.datasets")
|
||||
vllm_tokenizers = types.ModuleType("vllm.tokenizers")
|
||||
vllm_tokenizers_hf = types.ModuleType("vllm.tokenizers.hf")
|
||||
|
||||
vllm_benchmarks_datasets.BenchmarkDataset = BenchmarkDataset # type: ignore[attr-defined]
|
||||
vllm_benchmarks_datasets.SampleRequest = SampleRequest # type: ignore[attr-defined]
|
||||
vllm_tokenizers.TokenizerLike = TokenizerLike # type: ignore[attr-defined]
|
||||
vllm_tokenizers_hf.get_cached_tokenizer = get_cached_tokenizer # type: ignore[attr-defined]
|
||||
|
||||
sys.modules["vllm"] = vllm_mod
|
||||
sys.modules["vllm.benchmarks"] = vllm_benchmarks
|
||||
sys.modules["vllm.benchmarks.datasets"] = vllm_benchmarks_datasets
|
||||
sys.modules["vllm.tokenizers"] = vllm_tokenizers
|
||||
sys.modules["vllm.tokenizers.hf"] = vllm_tokenizers_hf
|
||||
|
||||
|
||||
# Install stubs immediately at collection time (before any test import).
|
||||
_install_vllm_stubs()
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# Add benchmarks/tts to path for import
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "benchmarks" / "tts"))
|
||||
import bench_tts
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model_configs_path(tmp_path: Path) -> Path:
|
||||
cfg = {
|
||||
"models": {
|
||||
"test/ModelA": {
|
||||
"stage_config": "model_a.yaml",
|
||||
"supported_tasks": ["voice_clone", "default_voice"],
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"task_extra_body": {
|
||||
"voice_clone": {"task_type": "Base"},
|
||||
"default_voice": {"voice": "Vivian", "task_type": "CustomVoice"},
|
||||
},
|
||||
},
|
||||
"test/ModelB": {
|
||||
"stage_config": "model_b.yaml",
|
||||
"supported_tasks": ["voice_clone"],
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"task_extra_body": {"voice_clone": {}},
|
||||
},
|
||||
}
|
||||
}
|
||||
p = tmp_path / "model_configs.yaml"
|
||||
p.write_text(yaml.dump(cfg), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def test_load_model_configs(model_configs_path: Path) -> None:
|
||||
configs = bench_tts.load_model_configs(model_configs_path)
|
||||
assert "test/ModelA" in configs
|
||||
assert "test/ModelB" in configs
|
||||
assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
|
||||
|
||||
|
||||
def test_build_bench_args_voice_clone(model_configs_path: Path) -> None:
|
||||
configs = bench_tts.load_model_configs(model_configs_path)
|
||||
cmd = bench_tts.build_bench_args(
|
||||
host="localhost",
|
||||
port=8000,
|
||||
model="test/ModelA",
|
||||
task="voice_clone",
|
||||
model_cfg=configs["test/ModelA"],
|
||||
locale="en",
|
||||
num_prompts=10,
|
||||
concurrency=1,
|
||||
dataset_path="/data/seed-tts",
|
||||
wer_eval=False,
|
||||
output_dir=None,
|
||||
result_filename=None,
|
||||
extra_cli_args=[],
|
||||
)
|
||||
assert "--dataset-name" in cmd
|
||||
idx = cmd.index("--dataset-name")
|
||||
assert cmd[idx + 1] == "seed-tts"
|
||||
assert "--max-concurrency" in cmd
|
||||
assert "--extra-body" in cmd
|
||||
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
|
||||
assert extra_body.get("task_type") == "Base"
|
||||
|
||||
|
||||
def test_build_bench_args_default_voice_has_voice_param(model_configs_path: Path) -> None:
|
||||
configs = bench_tts.load_model_configs(model_configs_path)
|
||||
cmd = bench_tts.build_bench_args(
|
||||
host="localhost",
|
||||
port=8000,
|
||||
model="test/ModelA",
|
||||
task="default_voice",
|
||||
model_cfg=configs["test/ModelA"],
|
||||
locale="en",
|
||||
num_prompts=10,
|
||||
concurrency=1,
|
||||
dataset_path="/data/seed-tts",
|
||||
wer_eval=False,
|
||||
output_dir=None,
|
||||
result_filename=None,
|
||||
extra_cli_args=[],
|
||||
)
|
||||
idx = cmd.index("--dataset-name")
|
||||
assert cmd[idx + 1] == "seed-tts-text"
|
||||
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
|
||||
assert extra_body.get("voice") == "Vivian"
|
||||
|
||||
|
||||
def test_build_bench_args_wer_eval_adds_flag(model_configs_path: Path) -> None:
|
||||
configs = bench_tts.load_model_configs(model_configs_path)
|
||||
cmd = bench_tts.build_bench_args(
|
||||
host="localhost",
|
||||
port=8000,
|
||||
model="test/ModelA",
|
||||
task="voice_clone",
|
||||
model_cfg=configs["test/ModelA"],
|
||||
locale="en",
|
||||
num_prompts=10,
|
||||
concurrency=1,
|
||||
dataset_path="/data/seed-tts",
|
||||
wer_eval=True,
|
||||
output_dir=None,
|
||||
result_filename=None,
|
||||
extra_cli_args=[],
|
||||
)
|
||||
assert "--seed-tts-wer-eval" in cmd
|
||||
|
||||
|
||||
def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.CaptureFixture, mocker) -> None:
|
||||
# ModelB does not support voice_design
|
||||
mocker.patch.object(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"bench_tts.py",
|
||||
"--model",
|
||||
"test/ModelB",
|
||||
"--task",
|
||||
"voice_design",
|
||||
"--model-configs",
|
||||
str(model_configs_path),
|
||||
],
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
bench_tts.main()
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
|
||||
and SeedTTSDesignSampleRequest.
|
||||
|
||||
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "seed_tts_dataset.py"
|
||||
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
||||
SeedTTSDesignDataset,
|
||||
SeedTTSDesignSampleRequest,
|
||||
SeedTTSTextDataset,
|
||||
SeedTTSTextSampleRequest,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def seed_tts_root(tmp_path: Path) -> Path:
|
||||
"""Minimal seed-tts-style directory with 5 entries."""
|
||||
locale_dir = tmp_path / "en"
|
||||
locale_dir.mkdir()
|
||||
wav_dir = locale_dir / "prompt-wavs"
|
||||
wav_dir.mkdir()
|
||||
for i in range(5):
|
||||
(wav_dir / f"utt{i:03d}.wav").write_bytes(b"RIFF\x00\x00\x00\x00WAVE")
|
||||
meta = "\n".join(f"utt{i:03d}|ref text {i}|prompt-wavs/utt{i:03d}.wav|target text {i}" for i in range(5))
|
||||
(locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
|
||||
return tmp_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mocker):
|
||||
ds = SeedTTSTextDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
tokenizer = mocker.MagicMock()
|
||||
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
|
||||
requests = ds.sample(tokenizer, num_requests=3)
|
||||
assert len(requests) == 3
|
||||
for req in requests:
|
||||
assert isinstance(req, SeedTTSTextSampleRequest)
|
||||
assert req.seed_tts_speech_extra is None or "ref_audio" not in (req.seed_tts_speech_extra or {})
|
||||
assert req.seed_tts_ref_wav_path == ""
|
||||
assert "target text" in req.prompt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SeedTTSDesignDataset tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def seed_tts_design_root(tmp_path: Path) -> Path:
|
||||
"""seed-tts-design directory with 5-field meta.lst entries."""
|
||||
locale_dir = tmp_path / "en"
|
||||
locale_dir.mkdir()
|
||||
meta = "\n".join(
|
||||
f"des{i:03d}|||target text {i}|A warm {['female', 'male'][i % 2]} voice with neutral accent." for i in range(5)
|
||||
)
|
||||
(locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_seed_tts_design_dataset_has_instructions(seed_tts_design_root, mocker):
|
||||
ds = SeedTTSDesignDataset(
|
||||
dataset_path=str(seed_tts_design_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
tokenizer = mocker.MagicMock()
|
||||
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
|
||||
requests = ds.sample(tokenizer, num_requests=3)
|
||||
assert len(requests) == 3
|
||||
for req in requests:
|
||||
assert isinstance(req, SeedTTSDesignSampleRequest)
|
||||
extra = req.seed_tts_speech_extra or {}
|
||||
assert "instructions" in extra
|
||||
assert extra["instructions"], "instructions must be non-empty"
|
||||
assert extra.get("task_type") == "VoiceDesign"
|
||||
assert "ref_audio" not in extra
|
||||
assert req.seed_tts_ref_wav_path == ""
|
||||
|
||||
|
||||
def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_root, mocker):
|
||||
"""Lines without a voice_description should be skipped."""
|
||||
locale_dir = seed_tts_design_root / "en"
|
||||
(locale_dir / "meta.lst").write_text(
|
||||
"bad001|||target text without description\nok001|||good target|A clear female voice.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
ds = SeedTTSDesignDataset(
|
||||
dataset_path=str(seed_tts_design_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
tokenizer = mocker.MagicMock()
|
||||
tokenizer.encode = lambda text, **kw: [0] * len(text.split())
|
||||
requests = ds.sample(tokenizer, num_requests=10)
|
||||
assert len(requests) == 1 # only the valid row
|
||||
|
||||
|
||||
def test_attach_sets_seed_tts_row_even_without_extra_body():
|
||||
"""seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
|
||||
|
||||
req = SeedTTSTextSampleRequest(
|
||||
prompt="hello world",
|
||||
prompt_len=2,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="test-0",
|
||||
seed_tts_speech_extra=None,
|
||||
seed_tts_ref_wav_path="",
|
||||
)
|
||||
assert req.seed_tts_speech_extra is None
|
||||
assert req.seed_tts_ref_wav_path == ""
|
||||
# The fix ensures that even with speech_extra=None, the function
|
||||
# sets seed_tts_row=True. We verify the source code has the fix.
|
||||
import inspect
|
||||
|
||||
import vllm_omni.benchmarks.patch.patch as patch_mod
|
||||
|
||||
src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
|
||||
# seed_tts_row must be set BEFORE the 'if not ex: return' check
|
||||
row_pos = src.index("seed_tts_row")
|
||||
not_ex_pos = src.index("if not ex:")
|
||||
assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"
|
||||
Reference in new issue
Block a user