diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index c08d3f333..130e9e59c 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -172,7 +172,7 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model' " agents: - queue: "gpu_1_queue" + queue: "gpu_4_queue" plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT @@ -193,7 +193,7 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' " agents: - queue: "gpu_1_queue" + queue: "gpu_4_queue" plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 66e03d5c7..5308957fb 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -388,7 +388,7 @@ steps: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: - nvidia.com/gpu: 1 + nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 0fc174d8b..a9aea9f62 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -303,7 +303,7 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' " agents: - queue: "gpu_1_queue" + queue: "gpu_4_queue" plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT @@ -326,7 +326,7 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model' " agents: - queue: "gpu_1_queue" + queue: "gpu_4_queue" plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT diff --git a/benchmarks/build_dataset/seed_tts_design/en/meta.lst b/benchmarks/build_dataset/seed_tts_design/en/meta.lst index 7e364c2e5..c1665f597 100644 --- a/benchmarks/build_dataset/seed_tts_design/en/meta.lst +++ b/benchmarks/build_dataset/seed_tts_design/en/meta.lst @@ -18,3 +18,13 @@ vd017|||The quarterly results exceed expectations across all major metrics.|A sh vd018|||Chapter one. The morning sun filtered gently through the forest canopy.|A smooth, rich male audiobook narrator voice, expressive and engaging. vd019|||To be or not to be, that is the question.|A theatrical female voice, dramatic and expressive, stage projection quality. vd020|||And that is all for tonight. Stay well out there, everyone.|A warm, velvety male late-night radio DJ voice, smooth and intimate. +vd021|||Please fasten your seatbelts; we will be experiencing some turbulence shortly.|A calm, professional female flight attendant voice, reassuring and clear. +vd022|||The fossil record suggests these creatures lived more than sixty million years ago.|A measured male documentary narrator voice, scientific and curious. +vd023|||Hey kiddo, did you finish your homework before dinner today?|A gentle, caring father voice, warm and slightly tired after a long day. +vd024|||Live from courtside, the home team has just taken a commanding lead.|A high-energy male sports broadcaster, urgent and engaged. +vd025|||Please remain on the line while I transfer you to the next available agent.|A polite, neutral female call-center voice, professional and patient. +vd026|||I have a dream that one day this nation will rise up and live out the true meaning of its creed.|A powerful, resonant male orator voice, slow and impassioned. +vd027|||Mama always said life was like a box of chocolates; you never know what you're gonna get.|A folksy male voice with a Southern American accent, warm and reflective. +vd028|||Ahoy there, matey! Hoist the colors and prepare to set sail.|A theatrical pirate male voice, gruff and boisterous. +vd029|||Ladies and gentlemen, please welcome to the stage our keynote speaker.|A polished female event MC voice, gracious and articulate. +vd030|||Once you've completed the form, click submit and we'll review it within two business days.|A friendly female product-onboarding voice, encouraging and clear. diff --git a/benchmarks/build_dataset/seed_tts_smoke/en/meta.lst b/benchmarks/build_dataset/seed_tts_smoke/en/meta.lst index afe4bc8ab..5c50e0c11 100644 --- a/benchmarks/build_dataset/seed_tts_smoke/en/meta.lst +++ b/benchmarks/build_dataset/seed_tts_smoke/en/meta.lst @@ -18,3 +18,13 @@ smoke017|||In the distance, the mountains were shrouded in thick morning fog. smoke018|||Our company reported record revenue for the fourth quarter of the fiscal year. smoke019|||She explained the new policy in detail during the staff meeting this morning. smoke020|||The children laughed and played in the garden until the sun began to set. +smoke021|||A long line of cars stretched along the coastal highway after the lighthouse opened. +smoke022|||The chef recommended the seared scallops with a lemon butter sauce as the special. +smoke023|||Our quarterly engineering review begins promptly at two in the afternoon on Thursday. +smoke024|||Despite the heavy rain, the marathon runners kept a steady pace through downtown. +smoke025|||She finished the crossword puzzle in under fifteen minutes while sipping her tea. +smoke026|||The new release adds streaming output, faster startup, and better error reporting. +smoke027|||His grandfather taught him how to whittle small wooden birds from cedar offcuts. +smoke028|||Travelers were advised to check the timetable since trains were running on a holiday schedule. +smoke029|||The startup launched a public beta after eighteen months of closed development. +smoke030|||At night the desert sky filled with stars so bright the Milky Way was clearly visible. diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index fbd09d219..ccf86e6ca 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -2,16 +2,19 @@ { "test_name": "test_qwen3_tts_base", "server_params": { - "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base" + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stage_overrides": { + "1": {"devices": "1"} + } }, "benchmark_params": [ { "task": "voice_clone", "eval_phase": "latency", - "enabled": false, "dataset_name": "seed-tts", "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", "num_prompts": [20], "max_concurrency": [1], "seed_tts_locale": "en", @@ -24,18 +27,18 @@ { "task": "voice_clone", "eval_phase": "throughput", - "enabled": false, "dataset_name": "seed-tts", "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", - "num_prompts": [80], - "max_concurrency": [8], + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [80, 128, 256], + "max_concurrency": [8, 64, 128], "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3500], - "median_audio_rtf": [0.75], - "audio_throughput": [10.0] + "median_audio_ttfp_ms": [3500, 7000, 12000], + "median_audio_rtf": [0.75, 1.5, 2.5], + "audio_throughput": [10.0, 40.0, 50.0] } }, { @@ -45,6 +48,7 @@ "dataset_name": "seed-tts", "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", "num_prompts": [200], "max_concurrency": [4], "seed_tts_locale": "en", @@ -59,7 +63,10 @@ { "test_name": "test_qwen3_tts_customvoice", "server_params": { - "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "stage_overrides": { + "1": {"devices": "1"} + } }, "benchmark_params": [ { @@ -86,15 +93,15 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [80], - "max_concurrency": [8], + "num_prompts": [80, 128, 256], + "max_concurrency": [8, 64, 128], "seed_tts_locale": "en", "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500], - "median_audio_rtf": [0.30], - "audio_throughput": [30.0] + "median_audio_ttfp_ms": [1500, 3000, 5500], + "median_audio_rtf": [0.30, 0.60, 1.0], + "audio_throughput": [30.0, 100.0, 120.0] } }, { @@ -157,15 +164,15 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [80], - "max_concurrency": [8], + "num_prompts": [80, 128, 256], + "max_concurrency": [8, 64, 128], "seed_tts_locale": "en", "extra_body": {"task_type": "VoiceDesign", "language": "English"}, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500], - "median_audio_rtf": [0.35], - "audio_throughput": [25.0] + "median_audio_ttfp_ms": [1500, 3500, 6000], + "median_audio_rtf": [0.35, 0.70, 1.2], + "audio_throughput": [25.0, 80.0, 100.0] } }, {