[CI/Build] Voxtral TTS Tests (#3738)

Signed-off-by: Clodagh Walsh <clodaghwalsh17@gmail.com>

examples/offline_inference/text_to_speech/voxtral_tts/end2end.py

@@ -212,7 +212,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir
     output_audio_dur = 0.0
 
     for batch_idx, o in enumerate(outputs):
-        audio_tensor = torch.cat(o.multimodal_output["audio"])
+        audio_tensor = torch.cat((o.multimodal_output["audio"],))
         audio_array = audio_tensor.tolist()
         output_audio_dur += float(len(audio_array)) / 24000
         if args.write_audio:

tests/dfx/perf/tests/test_tts.json

@@ -339,5 +339,66 @@
         }
       }
     ]
+  },
+  {
+    "test_name": "test_voxtral_tts",
+    "server_params": {
+      "model": "mistralai/Voxtral-4B-TTS-2603"
+    },
+    "benchmark_params": [
+      {
+        "task": "default_voice",
+        "eval_phase": "latency",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [20],
+        "max_concurrency": [1],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "casual_male"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [150],
+          "median_audio_rtf": [0.15]
+        }
+      },
+      {
+        "task": "default_voice",
+        "eval_phase": "throughput",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [80],
+        "max_concurrency": [8],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "casual_male"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [1500],
+          "median_audio_rtf": [0.30],
+          "audio_throughput": [30.0]
+        }
+      },
+      {
+        "task": "default_voice",
+        "eval_phase": "stress",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [100],
+        "request_rate": [2.0],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "casual_male"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [3000],
+          "median_audio_rtf": [0.25],
+          "audio_throughput": [4.0]
+        }
+      }
+    ]
   }
 ]

tests/e2e/online_serving/test_voxtral_tts.py

@@ -91,3 +91,90 @@ class TestVoxtralTTSFixedVoice:
                     "timeout": 120.0,
                 }
             )
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_speech_speed(self, omni_server, openai_client) -> None:
+        """Request with speed parameters"""
+        speeds = [0.5, 1, 1.5, 2, 2.5]
+        for speed in speeds:
+            openai_client.send_audio_speech_request(
+                {
+                    "model": omni_server.model,
+                    "input": "The boy was there when the sun rose.",
+                    "voice": "casual_female",
+                    "language": "English",
+                    "response_format": "wav",
+                    "timeout": 120.0,
+                    "speed": speed,
+                }
+            )
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_speech_instructions(self, omni_server, openai_client) -> None:
+        """Request with instructions parameters"""
+        instructions = [
+            "Speak formally",
+            "Speak angrily",
+            "Deliver with a sad voice",
+            "Speak with a chirpy happy voice",
+        ]
+        for instruction in instructions:
+            openai_client.send_audio_speech_request(
+                {
+                    "model": omni_server.model,
+                    "input": "The boy was there when the sun rose.",
+                    "voice": "casual_female",
+                    "language": "English",
+                    "response_format": "wav",
+                    "timeout": 120.0,
+                    "instructions": instruction,
+                }
+            )
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_speech_response_formats(self, omni_server, openai_client) -> None:
+        """Test TTS with different response formats"""
+        response_formats = ["wav", "mp3"]
+        for response_format in response_formats:
+            openai_client.send_audio_speech_request(
+                {
+                    "model": omni_server.model,
+                    "input": "Testing various response formats.",
+                    "voice": "casual_male",
+                    "language": "English",
+                    "response_format": response_format,
+                    "timeout": 120.0,
+                }
+            )
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_speech_batches(self, omni_server, openai_client) -> None:
+        """Test TTS batches"""
+        items = [
+            {"input": "The birch canoe slid on the smooth planks."},
+            {"input": "Glue the sheet to the dark blue background."},
+            {"input": "It's easy to tell the depth of a well."},
+            {"input": "These days a chicken leg is a rare dish."},
+            {"input": "Rice is often served in round bowls."},
+        ]
+
+        openai_client.send_audio_speech_batch_http_request(
+            {
+                "json": {
+                    "model": omni_server.model,
+                    "items": items,
+                    "voice": "casual_male",
+                    "language": "English",
+                    "response_format": "wav",
+                },
+                "timeout": 120.0,
+            }
+        )

tests/helpers/runtime.py

@@ -1767,7 +1767,16 @@ class OpenAIClientHandler:
         # Qwen3-TTS custom fields, forwarded via extra_body.
         extra_body: dict[str, Any] = {}
         # Keep this list aligned with vllm_omni.entrypoints.openai.protocol.audio params.
-        for key in ("task_type", "ref_text", "ref_audio", "language", "max_new_tokens", "seed"):
+        for key in (
+            "task_type",
+            "ref_text",
+            "ref_audio",
+            "language",
+            "max_new_tokens",
+            "seed",
+            "instructions",
+            "speed",
+        ):
             if key in request_config:
                 extra_body[key] = request_config[key]