[ci][nightly] Voxcpm2 performance benchmark (#3864)
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
ChenWenjingandGao Han authored and GitHub committed
1 parent
5dcf18a984
commit
834abdaf8b
6 files changed
+279
-3
No files matched your search
@@ -459,6 +459,52 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: VoxCPM2 · Perf Test"
|
||||
key: nightly-voxcpm2-performance
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
# Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
|
||||
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
||||
key: nightly-diffusion-x2iat-group
|
||||
|
||||
@@ -130,6 +130,9 @@ def build_bench_args(
|
||||
if task_extra_body:
|
||||
cmd += ["--extra-body", json.dumps(task_extra_body, separators=(",", ":"))]
|
||||
|
||||
if model_cfg.get("trust_remote_code"):
|
||||
cmd.append("--trust-remote-code")
|
||||
|
||||
if concurrency is not None:
|
||||
cmd += ["--max-concurrency", str(concurrency), "--request-rate", "inf"]
|
||||
|
||||
|
||||
@@ -32,8 +32,11 @@ models:
|
||||
task_type: Base
|
||||
|
||||
openbmb/VoxCPM2:
|
||||
supported_tasks: [voice_clone]
|
||||
supported_tasks: [voice_clone, default_voice]
|
||||
trust_remote_code: true
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
default_voice:
|
||||
voice: default
|
||||
@@ -191,7 +191,16 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
raise ValueError("The number of prompts does not match the QPS or max_concurrency")
|
||||
|
||||
args = ["--host", host, "--port", str(port)]
|
||||
exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
"baseline",
|
||||
"num_prompts",
|
||||
"max_concurrency",
|
||||
"task",
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
@@ -207,6 +216,14 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
elif not isinstance(value, bool):
|
||||
args.extend([arg_name, str(value)])
|
||||
|
||||
for config in BENCHMARK_CONFIGS:
|
||||
if config.get("test_name") != test_name:
|
||||
continue
|
||||
server_params = config.get("server_params") or {}
|
||||
if server_params.get("trust_remote_code") or params.get("trust_remote_code"):
|
||||
args.append("--trust-remote-code")
|
||||
break
|
||||
|
||||
# QPS / request-rate sweep
|
||||
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
|
||||
@@ -17,7 +17,16 @@ def test_task_excluded_from_cli_args():
|
||||
"percentile-metrics": "audio_rtf,audio_ttfp",
|
||||
"baseline": {"mean_audio_rtf": [0.5]},
|
||||
}
|
||||
exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
"baseline",
|
||||
"num_prompts",
|
||||
"max_concurrency",
|
||||
"task",
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
}
|
||||
args = []
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_voxcpm2",
|
||||
"server_params": {
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"trust_remote_code": true
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "latency",
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
600
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.2
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "throughput",
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
80
|
||||
],
|
||||
"max_concurrency": [
|
||||
8
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
3500
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.85
|
||||
],
|
||||
"audio_throughput": [
|
||||
8
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "voice_clone",
|
||||
"eval_phase": "quality",
|
||||
"enabled": false,
|
||||
"dataset_name": "seed-tts",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "linyueqian/seed-tts-eval-subset",
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "latency",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
20
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
350
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.15
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "throughput",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
80
|
||||
],
|
||||
"max_concurrency": [
|
||||
8
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
3000
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5
|
||||
],
|
||||
"audio_throughput": [
|
||||
10
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "stress",
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
2.0
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"median_audio_ttfp_ms": [
|
||||
500
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3
|
||||
],
|
||||
"audio_throughput": [
|
||||
5
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"task": "default_voice",
|
||||
"eval_phase": "quality",
|
||||
"enabled": false,
|
||||
"dataset_name": "seed-tts-text",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||
"num_prompts": [
|
||||
200
|
||||
],
|
||||
"max_concurrency": [
|
||||
4
|
||||
],
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"voice": "default"
|
||||
},
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user