[ci][nightly] Voxcpm2 performance benchmark (#3864)
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
5dcf18a984
commit
834abdaf8b
6 files changed
+279
-3
No files matched your search
@@ -191,7 +191,16 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
raise ValueError("The number of prompts does not match the QPS or max_concurrency")
|
||||
|
||||
args = ["--host", host, "--port", str(port)]
|
||||
exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
"baseline",
|
||||
"num_prompts",
|
||||
"max_concurrency",
|
||||
"task",
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
@@ -207,6 +216,14 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
elif not isinstance(value, bool):
|
||||
args.extend([arg_name, str(value)])
|
||||
|
||||
for config in BENCHMARK_CONFIGS:
|
||||
if config.get("test_name") != test_name:
|
||||
continue
|
||||
server_params = config.get("server_params") or {}
|
||||
if server_params.get("trust_remote_code") or params.get("trust_remote_code"):
|
||||
args.append("--trust-remote-code")
|
||||
break
|
||||
|
||||
# QPS / request-rate sweep
|
||||
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
|
||||
@@ -17,7 +17,16 @@ def test_task_excluded_from_cli_args():
|
||||
"percentile-metrics": "audio_rtf,audio_ttfp",
|
||||
"baseline": {"mean_audio_rtf": [0.5]},
|
||||
}
|
||||
exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
"baseline",
|
||||
"num_prompts",
|
||||
"max_concurrency",
|
||||
"task",
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
}
|
||||
args = []
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_voxcpm2",
|
||||
"server_params": {
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"trust_remote_code": true
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "latency",
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
600
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.2
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "throughput",
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
80
|
||||
],
|
||||
"max_concurrency": [
|
||||
8
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
3500
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.85
|
||||
],
|
||||
"audio_throughput": [
|
||||
8
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "quality",
|
||||
"enabled": false,
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "latency",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
350
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.15
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "throughput",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
80
|
||||
],
|
||||
"max_concurrency": [
|
||||
8
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
3000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5
|
||||
],
|
||||
"audio_throughput": [
|
||||
10
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "stress",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
2.0
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
500
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3
|
||||
],
|
||||
"audio_throughput": [
|
||||
5
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "quality",
|
||||
"enabled": false,
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user