405 lines
10 KiB
JSON
405 lines
10 KiB
JSON
[
|
|
{
|
|
"test_name": "test_qwen3_tts_base",
|
|
"server_params": {
|
|
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
|
"stage_overrides": {
|
|
"1": {
|
|
"devices": "1"
|
|
}
|
|
}
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
3000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.6
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
1000,
|
|
3000,
|
|
14000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.70,
|
|
1.15,
|
|
3.90
|
|
],
|
|
"audio_throughput": [
|
|
12,
|
|
14,
|
|
14
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "quality",
|
|
"enabled": false,
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
200
|
|
],
|
|
"max_concurrency": [
|
|
4
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"seed_tts_wer_eval": true,
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"mean_audio_rtf": [
|
|
0.45
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"test_name": "test_qwen3_tts_customvoice",
|
|
"server_params": {
|
|
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
|
"stage_overrides": {
|
|
"1": {
|
|
"devices": "1"
|
|
}
|
|
}
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
2000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
300,
|
|
1000,
|
|
5000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.25,
|
|
0.35,
|
|
1.10
|
|
],
|
|
"audio_throughput": [
|
|
30,
|
|
40,
|
|
40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "quality",
|
|
"enabled": false,
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
200
|
|
],
|
|
"max_concurrency": [
|
|
4
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"seed_tts_wer_eval": true,
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"mean_audio_rtf": [
|
|
0.35
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "stress",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
2.0
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.20
|
|
],
|
|
"audio_throughput": [
|
|
10
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.15
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100,
|
|
1000,
|
|
5000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.25,
|
|
0.40,
|
|
1.20
|
|
],
|
|
"audio_throughput": [
|
|
30,
|
|
40,
|
|
40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "stress",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
2.0
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.20
|
|
],
|
|
"audio_throughput": [
|
|
9
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"test_name": "test_voxtral_tts",
|
|
"server_params": {
|
|
"model": "mistralai/Voxtral-4B-TTS-2603"
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [20],
|
|
"max_concurrency": [1],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {"voice": "casual_male"},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [150],
|
|
"median_audio_rtf": [0.15]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [80],
|
|
"max_concurrency": [8],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {"voice": "casual_male"},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [1500],
|
|
"median_audio_rtf": [0.30],
|
|
"audio_throughput": [30.0]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "stress",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [100],
|
|
"request_rate": [2.0],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {"voice": "casual_male"},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [3000],
|
|
"median_audio_rtf": [0.25],
|
|
"audio_throughput": [4.0]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|