wangyuandHongsheng Liu
4e82192fd6
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
344 lines
8.3 KiB
JSON
344 lines
8.3 KiB
JSON
[
|
|
{
|
|
"test_name": "test_qwen3_tts_base",
|
|
"server_params": {
|
|
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
|
"stage_overrides": {
|
|
"1": {
|
|
"devices": "1"
|
|
}
|
|
}
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
3000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.6
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
1000,
|
|
3000,
|
|
14000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.70,
|
|
1.15,
|
|
3.90
|
|
],
|
|
"audio_throughput": [
|
|
12,
|
|
14,
|
|
14
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_clone",
|
|
"eval_phase": "quality",
|
|
"enabled": false,
|
|
"dataset_name": "seed-tts",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
|
"num_prompts": [
|
|
200
|
|
],
|
|
"max_concurrency": [
|
|
4
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"seed_tts_wer_eval": true,
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"mean_audio_rtf": [
|
|
0.45
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"test_name": "test_qwen3_tts_customvoice",
|
|
"server_params": {
|
|
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
|
"stage_overrides": {
|
|
"1": {
|
|
"devices": "1"
|
|
}
|
|
}
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
2000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
300,
|
|
1000,
|
|
5000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.25,
|
|
0.35,
|
|
1.10
|
|
],
|
|
"audio_throughput": [
|
|
30,
|
|
40,
|
|
40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "quality",
|
|
"enabled": false,
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
200
|
|
],
|
|
"max_concurrency": [
|
|
4
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"seed_tts_wer_eval": true,
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"mean_audio_rtf": [
|
|
0.35
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "default_voice",
|
|
"eval_phase": "stress",
|
|
"dataset_name": "seed-tts-text",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
2.0
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"voice": "Vivian",
|
|
"language": "English",
|
|
"task_type": "CustomVoice"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.20
|
|
],
|
|
"audio_throughput": [
|
|
10
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "latency",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
20
|
|
],
|
|
"max_concurrency": [
|
|
1
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.15
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "throughput",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
80,
|
|
128,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
8,
|
|
16,
|
|
64
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100,
|
|
1000,
|
|
5000
|
|
],
|
|
"median_audio_rtf": [
|
|
0.25,
|
|
0.40,
|
|
1.20
|
|
],
|
|
"audio_throughput": [
|
|
30,
|
|
40,
|
|
40
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"task": "voice_design",
|
|
"eval_phase": "stress",
|
|
"dataset_name": "seed-tts-design",
|
|
"backend": "openai-audio-speech",
|
|
"endpoint": "/v1/audio/speech",
|
|
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
2.0
|
|
],
|
|
"seed_tts_locale": "en",
|
|
"extra_body": {
|
|
"task_type": "VoiceDesign",
|
|
"language": "English"
|
|
},
|
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"median_audio_ttfp_ms": [
|
|
100
|
|
],
|
|
"median_audio_rtf": [
|
|
0.20
|
|
],
|
|
"audio_throughput": [
|
|
9
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|