From aa92a3a35d45027d9ca5eb2ec7ef7e27dca1a843 Mon Sep 17 00:00:00 2001 From: Clodagh Walsh <42538837+clodaghwalsh17@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:32:21 +0100 Subject: [PATCH] [CI/Build] Voxtral TTS Tests (#3738) Signed-off-by: Clodagh Walsh --- .../text_to_speech/voxtral_tts/end2end.py | 2 +- tests/dfx/perf/tests/test_tts.json | 61 +++++++++++++ tests/e2e/online_serving/test_voxtral_tts.py | 87 +++++++++++++++++++ tests/helpers/runtime.py | 11 ++- 4 files changed, 159 insertions(+), 2 deletions(-) diff --git a/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py b/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py index c740197b5..c9225c212 100644 --- a/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/voxtral_tts/end2end.py @@ -212,7 +212,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir output_audio_dur = 0.0 for batch_idx, o in enumerate(outputs): - audio_tensor = torch.cat(o.multimodal_output["audio"]) + audio_tensor = torch.cat((o.multimodal_output["audio"],)) audio_array = audio_tensor.tolist() output_audio_dur += float(len(audio_array)) / 24000 if args.write_audio: diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 733c40963..9246b502f 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -339,5 +339,66 @@ } } ] + }, + { + "test_name": "test_voxtral_tts", + "server_params": { + "model": "mistralai/Voxtral-4B-TTS-2603" + }, + "benchmark_params": [ + { + "task": "default_voice", + "eval_phase": "latency", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [20], + "max_concurrency": [1], + "seed_tts_locale": "en", + "extra_body": {"voice": "casual_male"}, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [150], + "median_audio_rtf": [0.15] + } + }, + { + "task": "default_voice", + "eval_phase": "throughput", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [80], + "max_concurrency": [8], + "seed_tts_locale": "en", + "extra_body": {"voice": "casual_male"}, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [1500], + "median_audio_rtf": [0.30], + "audio_throughput": [30.0] + } + }, + { + "task": "default_voice", + "eval_phase": "stress", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [100], + "request_rate": [2.0], + "seed_tts_locale": "en", + "extra_body": {"voice": "casual_male"}, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "median_audio_ttfp_ms": [3000], + "median_audio_rtf": [0.25], + "audio_throughput": [4.0] + } + } + ] } ] diff --git a/tests/e2e/online_serving/test_voxtral_tts.py b/tests/e2e/online_serving/test_voxtral_tts.py index a1b5fdbb8..a86de2100 100644 --- a/tests/e2e/online_serving/test_voxtral_tts.py +++ b/tests/e2e/online_serving/test_voxtral_tts.py @@ -91,3 +91,90 @@ class TestVoxtralTTSFixedVoice: "timeout": 120.0, } ) + + @pytest.mark.advanced_model + @pytest.mark.tts + @hardware_test(res={"cuda": "H100"}, num_cards=1) + def test_speech_speed(self, omni_server, openai_client) -> None: + """Request with speed parameters""" + speeds = [0.5, 1, 1.5, 2, 2.5] + for speed in speeds: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "The boy was there when the sun rose.", + "voice": "casual_female", + "language": "English", + "response_format": "wav", + "timeout": 120.0, + "speed": speed, + } + ) + + @pytest.mark.advanced_model + @pytest.mark.tts + @hardware_test(res={"cuda": "H100"}, num_cards=1) + def test_speech_instructions(self, omni_server, openai_client) -> None: + """Request with instructions parameters""" + instructions = [ + "Speak formally", + "Speak angrily", + "Deliver with a sad voice", + "Speak with a chirpy happy voice", + ] + for instruction in instructions: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "The boy was there when the sun rose.", + "voice": "casual_female", + "language": "English", + "response_format": "wav", + "timeout": 120.0, + "instructions": instruction, + } + ) + + @pytest.mark.advanced_model + @pytest.mark.tts + @hardware_test(res={"cuda": "H100"}, num_cards=1) + def test_speech_response_formats(self, omni_server, openai_client) -> None: + """Test TTS with different response formats""" + response_formats = ["wav", "mp3"] + for response_format in response_formats: + openai_client.send_audio_speech_request( + { + "model": omni_server.model, + "input": "Testing various response formats.", + "voice": "casual_male", + "language": "English", + "response_format": response_format, + "timeout": 120.0, + } + ) + + @pytest.mark.advanced_model + @pytest.mark.tts + @hardware_test(res={"cuda": "H100"}, num_cards=1) + def test_speech_batches(self, omni_server, openai_client) -> None: + """Test TTS batches""" + items = [ + {"input": "The birch canoe slid on the smooth planks."}, + {"input": "Glue the sheet to the dark blue background."}, + {"input": "It's easy to tell the depth of a well."}, + {"input": "These days a chicken leg is a rare dish."}, + {"input": "Rice is often served in round bowls."}, + ] + + openai_client.send_audio_speech_batch_http_request( + { + "json": { + "model": omni_server.model, + "items": items, + "voice": "casual_male", + "language": "English", + "response_format": "wav", + }, + "timeout": 120.0, + } + ) diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 807ffb969..e07e9d7f9 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -1767,7 +1767,16 @@ class OpenAIClientHandler: # Qwen3-TTS custom fields, forwarded via extra_body. extra_body: dict[str, Any] = {} # Keep this list aligned with vllm_omni.entrypoints.openai.protocol.audio params. - for key in ("task_type", "ref_text", "ref_audio", "language", "max_new_tokens", "seed"): + for key in ( + "task_type", + "ref_text", + "ref_audio", + "language", + "max_new_tokens", + "seed", + "instructions", + "speed", + ): if key in request_config: extra_body[key] = request_config[key]