[Benchmark] Universal TTS benchmark: Qwen3-TTS + VoxCPM2 with 3 task types (voice-clone/default/design) (#2835)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>

tests/benchmarks/test_bench_tts_cli.py

@@ -0,0 +1,139 @@
+"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+# Add benchmarks/tts to path for import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "benchmarks" / "tts"))
+import bench_tts
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+@pytest.fixture()
+def model_configs_path(tmp_path: Path) -> Path:
+    cfg = {
+        "models": {
+            "test/ModelA": {
+                "stage_config": "model_a.yaml",
+                "supported_tasks": ["voice_clone", "default_voice"],
+                "backend": "openai-audio-speech",
+                "endpoint": "/v1/audio/speech",
+                "task_extra_body": {
+                    "voice_clone": {"task_type": "Base"},
+                    "default_voice": {"voice": "Vivian", "task_type": "CustomVoice"},
+                },
+            },
+            "test/ModelB": {
+                "stage_config": "model_b.yaml",
+                "supported_tasks": ["voice_clone"],
+                "backend": "openai-audio-speech",
+                "endpoint": "/v1/audio/speech",
+                "task_extra_body": {"voice_clone": {}},
+            },
+        }
+    }
+    p = tmp_path / "model_configs.yaml"
+    p.write_text(yaml.dump(cfg), encoding="utf-8")
+    return p
+
+
+def test_load_model_configs(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    assert "test/ModelA" in configs
+    assert "test/ModelB" in configs
+    assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
+
+
+def test_build_bench_args_voice_clone(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="voice_clone",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=False,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    assert "--dataset-name" in cmd
+    idx = cmd.index("--dataset-name")
+    assert cmd[idx + 1] == "seed-tts"
+    assert "--max-concurrency" in cmd
+    assert "--extra-body" in cmd
+    extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
+    assert extra_body.get("task_type") == "Base"
+
+
+def test_build_bench_args_default_voice_has_voice_param(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="default_voice",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=False,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    idx = cmd.index("--dataset-name")
+    assert cmd[idx + 1] == "seed-tts-text"
+    extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
+    assert extra_body.get("voice") == "Vivian"
+
+
+def test_build_bench_args_wer_eval_adds_flag(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="voice_clone",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=True,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    assert "--seed-tts-wer-eval" in cmd
+
+
+def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.CaptureFixture, mocker) -> None:
+    # ModelB does not support voice_design
+    mocker.patch.object(
+        sys,
+        "argv",
+        [
+            "bench_tts.py",
+            "--model",
+            "test/ModelB",
+            "--task",
+            "voice_design",
+            "--model-configs",
+            str(model_configs_path),
+        ],
+    )
+    with pytest.raises(SystemExit):
+        bench_tts.main()