From 0e2d55f46d81dea450e67155171ebd792518a36d Mon Sep 17 00:00:00 2001 From: ChenWenjing <54166744+Shirley125@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:11:04 +0800 Subject: [PATCH] [ci] add Voxcpm2 accuracy tests (#4055) Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> --- .buildkite/test-merge.yml | 23 +++++ .buildkite/test-nightly.yml | 30 +++++- .buildkite/test-ready.yml | 23 +++++ tests/e2e/online_serving/test_voxcpm2_tts.py | 97 +++++++++++++++++++ .../test_voxcpm2_tts_expansion.py | 95 ++++++++++++++++++ 5 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/online_serving/test_voxcpm2_tts.py create mode 100644 tests/e2e/online_serving/test_voxcpm2_tts_expansion.py diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 8d47539d3..191b7f214 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -389,6 +389,29 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "TTS · VoxCPM2 Online Test" + timeout_in_minutes: 30 + commands: + - | + timeout 30m bash -c " + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' + " + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Omni · Higgs-Audio V2 Test" timeout_in_minutes: 30 commands: diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 9a9840432..780db14cd 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -398,7 +398,7 @@ steps: timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy + - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py agents: queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -413,6 +413,30 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" + timeout_in_minutes: 30 + commands: + - | + timeout 30m bash -c " + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' + " + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: ":full_moon: TTS · Perf Test" key: nightly-tts-performance timeout_in_minutes: 180 @@ -459,8 +483,8 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: VoxCPM2 · Perf Test" - key: nightly-voxcpm2-performance + - label: ":full_moon: TTS · VoxCPM2 · Perf Test" + key: nightly-tts-performance-voxcpm2 timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 055b9b0e3..2e28db312 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -296,6 +296,29 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "TTS · VoxCPM2 Online Test" + timeout_in_minutes: 30 + commands: + - | + timeout 30m bash -c " + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model' + " + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "TTS · Qwen3-TTS CustomVoice Test" commands: - | diff --git a/tests/e2e/online_serving/test_voxcpm2_tts.py b/tests/e2e/online_serving/test_voxcpm2_tts.py new file mode 100644 index 000000000..cf493bc14 --- /dev/null +++ b/tests/e2e/online_serving/test_voxcpm2_tts.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E Online tests for VoxCPM2 with text input and audio output. + +These tests verify the /v1/audio/speech endpoint works correctly with +actual model inference, not mocks. VoxCPM2 zero-shot needs only ``input`` +and a placeholder ``voice`` (``default``); no ``task_type`` / ``ref_text``. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +MODEL = "openbmb/VoxCPM2" +DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0 +MAX_CONCURRENT = 4 + +# ~0.5 s of 48 kHz mono PCM_16 in WAV (~48k payload + header). +_MIN_AUDIO_BYTES = 40_000 + + +def get_prompt(prompt_type="text"): + """English prompt for zero-shot TTS (Whisper-friendly for advanced_model accuracy).""" + prompts = { + "text": "The weather is nice today, perfect for a walk in the park.", + } + return prompts.get(prompt_type, prompts["text"]) + + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("voxcpm2.yaml"), + server_args=["--trust-remote-code", "--disable-log-stats"], + ), + id="voxcpm2", + ) +] + + +@pytest.mark.advanced_model +@pytest.mark.core_model +@pytest.mark.tts +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_audio_001(omni_server, openai_client) -> None: + """ + Test zero-shot text-to-audio via OpenAI API. + Deploy Setting: voxcpm2.yaml + Input Modal: text + Output Modal: audio + Input Setting: stream=False + Datasets: few requests (max_num_seqs=4) + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": False, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "response_format": "wav", + "voice": "default", + "min_audio_bytes": _MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) + + +@pytest.mark.advanced_model +@pytest.mark.tts +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_audio_002(omni_server, openai_client) -> None: + """ + Test zero-shot streaming text-to-audio via OpenAI API. + Deploy Setting: voxcpm2.yaml + Input Modal: text + Output Modal: audio + Input Setting: stream=True + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": True, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "response_format": "wav", + "voice": "default", + "min_audio_bytes": _MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config) diff --git a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py new file mode 100644 index 000000000..b889a9e57 --- /dev/null +++ b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E Online expansion tests for VoxCPM2 (voice clone, PCM, concurrency). + +Voice clone uses ``ref_audio`` only (no ``ref_text``). Reference clip is +vendored under tests/assets/qwen3_tts/clone_2.wav — see test_qwen3_tts_base.py. +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.media import load_test_audio_data_url +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.full_model, pytest.mark.tts] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "openbmb/VoxCPM2" +DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0 +_MIN_AUDIO_BYTES = 40_000 +MAX_CONCURRENT = 4 + +REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav") + + +def get_prompt(prompt_type="text"): + prompts = { + "text": "The weather is nice today, perfect for a walk in the park.", + } + return prompts.get(prompt_type, prompts["text"]) + + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("voxcpm2.yaml"), + server_args=["--trust-remote-code", "--disable-log-stats"], + ), + id="voxcpm2", + ) +] + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_voice_clone_streaming_001(omni_server, openai_client) -> None: + """ + Test voice-clone text-to-audio via OpenAI API. + Deploy Setting: voxcpm2.yaml + Input Modal: text + ref_audio + Output Modal: audio + Input Setting: stream=True + Datasets: few requests (max_num_seqs=4) + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": True, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "response_format": "wav", + "voice": "default", + "ref_audio": REF_AUDIO_URL, + "min_audio_bytes": _MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_response_format_001(omni_server, openai_client) -> None: + """ + Test voice-clone non-stream PCM output via OpenAI API. + Deploy Setting: voxcpm2.yaml + Input Modal: text + ref_audio + Output Modal: audio (pcm) + Input Setting: stream=False + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "response_format": "pcm", + "stream": False, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "voice": "default", + "ref_audio": REF_AUDIO_URL, + "min_audio_bytes": _MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config)