diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 8a43539d4..9d13d8d74 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -459,6 +459,52 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: VoxCPM2 ยท Perf Test" + key: nightly-voxcpm2-performance + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below. - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" key: nightly-diffusion-x2iat-group diff --git a/benchmarks/tts/bench_tts.py b/benchmarks/tts/bench_tts.py index 8d2eb99c8..78608e844 100644 --- a/benchmarks/tts/bench_tts.py +++ b/benchmarks/tts/bench_tts.py @@ -130,6 +130,9 @@ def build_bench_args( if task_extra_body: cmd += ["--extra-body", json.dumps(task_extra_body, separators=(",", ":"))] + if model_cfg.get("trust_remote_code"): + cmd.append("--trust-remote-code") + if concurrency is not None: cmd += ["--max-concurrency", str(concurrency), "--request-rate", "inf"] diff --git a/benchmarks/tts/model_configs.yaml b/benchmarks/tts/model_configs.yaml index 83b253705..17254e37c 100644 --- a/benchmarks/tts/model_configs.yaml +++ b/benchmarks/tts/model_configs.yaml @@ -32,8 +32,11 @@ models: task_type: Base openbmb/VoxCPM2: - supported_tasks: [voice_clone] + supported_tasks: [voice_clone, default_voice] + trust_remote_code: true backend: openai-audio-speech endpoint: /v1/audio/speech task_extra_body: voice_clone: {} + default_voice: + voice: default diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 55ad2f8b2..7ca376323 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -191,7 +191,16 @@ def test_performance_benchmark(omni_server, benchmark_params, request): raise ValueError("The number of prompts does not match the QPS or max_concurrency") args = ["--host", host, "--port", str(port)] - exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"} + exclude_keys = { + "request_rate", + "baseline", + "num_prompts", + "max_concurrency", + "task", + "enabled", + "eval_phase", + "trust_remote_code", + } for key, value in params.items(): if key in exclude_keys or value is None: @@ -207,6 +216,14 @@ def test_performance_benchmark(omni_server, benchmark_params, request): elif not isinstance(value, bool): args.extend([arg_name, str(value)]) + for config in BENCHMARK_CONFIGS: + if config.get("test_name") != test_name: + continue + server_params = config.get("server_params") or {} + if server_params.get("trust_remote_code") or params.get("trust_remote_code"): + args.append("--trust-remote-code") + break + # QPS / request-rate sweep for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index 1276a8470..a195c4546 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -17,7 +17,16 @@ def test_task_excluded_from_cli_args(): "percentile-metrics": "audio_rtf,audio_ttfp", "baseline": {"mean_audio_rtf": [0.5]}, } - exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"} + exclude_keys = { + "request_rate", + "baseline", + "num_prompts", + "max_concurrency", + "task", + "enabled", + "eval_phase", + "trust_remote_code", + } args = [] for key, value in params.items(): if key in exclude_keys or value is None: diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json new file mode 100644 index 000000000..f19f95960 --- /dev/null +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -0,0 +1,198 @@ +[ + { + "test_name": "test_voxcpm2", + "server_params": { + "model": "openbmb/VoxCPM2", + "trust_remote_code": true + }, + "benchmark_params": [ + { + "task": "voice_clone", + "eval_phase": "latency", + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [ + 600 + ], + "median_audio_rtf": [ + 0.2 + ] + } + }, + { + "task": "voice_clone", + "eval_phase": "throughput", + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 80 + ], + "max_concurrency": [ + 8 + ], + "seed_tts_locale": "en", + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [ + 3500 + ], + "median_audio_rtf": [ + 0.85 + ], + "audio_throughput": [ + 8 + ] + } + }, + { + "task": "voice_clone", + "eval_phase": "quality", + "enabled": false, + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], + "seed_tts_locale": "en", + "seed_tts_wer_eval": true, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_audio_rtf": [ + 0.45 + ] + } + }, + { + "task": "default_voice", + "eval_phase": "latency", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "default" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [ + 350 + ], + "median_audio_rtf": [ + 0.15 + ] + } + }, + { + "task": "default_voice", + "eval_phase": "throughput", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 80 + ], + "max_concurrency": [ + 8 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "default" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [ + 3000 + ], + "median_audio_rtf": [ + 0.5 + ], + "audio_throughput": [ + 10 + ] + } + }, + { + "task": "default_voice", + "eval_phase": "stress", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "default" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [ + 500 + ], + "median_audio_rtf": [ + 0.3 + ], + "audio_throughput": [ + 5 + ] + } + }, + { + "task": "default_voice", + "eval_phase": "quality", + "enabled": false, + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "default" + }, + "seed_tts_wer_eval": true, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_audio_rtf": [ + 0.35 + ] + } + } + ] + } +]