[CI/Build] Voxtral TTS Tests (#3738)
Signed-off-by: Clodagh Walsh <clodaghwalsh17@gmail.com>
This commit is contained in:
1 parent
2fa4c7a4a9
commit
aa92a3a35d
4 files changed
+159
-2
No files matched your search
@@ -212,7 +212,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir
|
|||||||
output_audio_dur = 0.0
|
output_audio_dur = 0.0
|
||||||
|
|
||||||
for batch_idx, o in enumerate(outputs):
|
for batch_idx, o in enumerate(outputs):
|
||||||
audio_tensor = torch.cat(o.multimodal_output["audio"])
|
audio_tensor = torch.cat((o.multimodal_output["audio"],))
|
||||||
audio_array = audio_tensor.tolist()
|
audio_array = audio_tensor.tolist()
|
||||||
output_audio_dur += float(len(audio_array)) / 24000
|
output_audio_dur += float(len(audio_array)) / 24000
|
||||||
if args.write_audio:
|
if args.write_audio:
|
||||||
|
|||||||
@@ -339,5 +339,66 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"test_name": "test_voxtral_tts",
|
||||||
|
"server_params": {
|
||||||
|
"model": "mistralai/Voxtral-4B-TTS-2603"
|
||||||
|
},
|
||||||
|
"benchmark_params": [
|
||||||
|
{
|
||||||
|
"task": "default_voice",
|
||||||
|
"eval_phase": "latency",
|
||||||
|
"dataset_name": "seed-tts-text",
|
||||||
|
"backend": "openai-audio-speech",
|
||||||
|
"endpoint": "/v1/audio/speech",
|
||||||
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||||
|
"num_prompts": [20],
|
||||||
|
"max_concurrency": [1],
|
||||||
|
"seed_tts_locale": "en",
|
||||||
|
"extra_body": {"voice": "casual_male"},
|
||||||
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
|
"baseline": {
|
||||||
|
"median_audio_ttfp_ms": [150],
|
||||||
|
"median_audio_rtf": [0.15]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"task": "default_voice",
|
||||||
|
"eval_phase": "throughput",
|
||||||
|
"dataset_name": "seed-tts-text",
|
||||||
|
"backend": "openai-audio-speech",
|
||||||
|
"endpoint": "/v1/audio/speech",
|
||||||
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||||
|
"num_prompts": [80],
|
||||||
|
"max_concurrency": [8],
|
||||||
|
"seed_tts_locale": "en",
|
||||||
|
"extra_body": {"voice": "casual_male"},
|
||||||
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
|
"baseline": {
|
||||||
|
"median_audio_ttfp_ms": [1500],
|
||||||
|
"median_audio_rtf": [0.30],
|
||||||
|
"audio_throughput": [30.0]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"task": "default_voice",
|
||||||
|
"eval_phase": "stress",
|
||||||
|
"dataset_name": "seed-tts-text",
|
||||||
|
"backend": "openai-audio-speech",
|
||||||
|
"endpoint": "/v1/audio/speech",
|
||||||
|
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
|
||||||
|
"num_prompts": [100],
|
||||||
|
"request_rate": [2.0],
|
||||||
|
"seed_tts_locale": "en",
|
||||||
|
"extra_body": {"voice": "casual_male"},
|
||||||
|
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||||
|
"baseline": {
|
||||||
|
"median_audio_ttfp_ms": [3000],
|
||||||
|
"median_audio_rtf": [0.25],
|
||||||
|
"audio_throughput": [4.0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
@@ -91,3 +91,90 @@ class TestVoxtralTTSFixedVoice:
|
|||||||
"timeout": 120.0,
|
"timeout": 120.0,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_speech_speed(self, omni_server, openai_client) -> None:
|
||||||
|
"""Request with speed parameters"""
|
||||||
|
speeds = [0.5, 1, 1.5, 2, 2.5]
|
||||||
|
for speed in speeds:
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "The boy was there when the sun rose.",
|
||||||
|
"voice": "casual_female",
|
||||||
|
"language": "English",
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": 120.0,
|
||||||
|
"speed": speed,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_speech_instructions(self, omni_server, openai_client) -> None:
|
||||||
|
"""Request with instructions parameters"""
|
||||||
|
instructions = [
|
||||||
|
"Speak formally",
|
||||||
|
"Speak angrily",
|
||||||
|
"Deliver with a sad voice",
|
||||||
|
"Speak with a chirpy happy voice",
|
||||||
|
]
|
||||||
|
for instruction in instructions:
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "The boy was there when the sun rose.",
|
||||||
|
"voice": "casual_female",
|
||||||
|
"language": "English",
|
||||||
|
"response_format": "wav",
|
||||||
|
"timeout": 120.0,
|
||||||
|
"instructions": instruction,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_speech_response_formats(self, omni_server, openai_client) -> None:
|
||||||
|
"""Test TTS with different response formats"""
|
||||||
|
response_formats = ["wav", "mp3"]
|
||||||
|
for response_format in response_formats:
|
||||||
|
openai_client.send_audio_speech_request(
|
||||||
|
{
|
||||||
|
"model": omni_server.model,
|
||||||
|
"input": "Testing various response formats.",
|
||||||
|
"voice": "casual_male",
|
||||||
|
"language": "English",
|
||||||
|
"response_format": response_format,
|
||||||
|
"timeout": 120.0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.tts
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
|
def test_speech_batches(self, omni_server, openai_client) -> None:
|
||||||
|
"""Test TTS batches"""
|
||||||
|
items = [
|
||||||
|
{"input": "The birch canoe slid on the smooth planks."},
|
||||||
|
{"input": "Glue the sheet to the dark blue background."},
|
||||||
|
{"input": "It's easy to tell the depth of a well."},
|
||||||
|
{"input": "These days a chicken leg is a rare dish."},
|
||||||
|
{"input": "Rice is often served in round bowls."},
|
||||||
|
]
|
||||||
|
|
||||||
|
openai_client.send_audio_speech_batch_http_request(
|
||||||
|
{
|
||||||
|
"json": {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"items": items,
|
||||||
|
"voice": "casual_male",
|
||||||
|
"language": "English",
|
||||||
|
"response_format": "wav",
|
||||||
|
},
|
||||||
|
"timeout": 120.0,
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -1767,7 +1767,16 @@ class OpenAIClientHandler:
|
|||||||
# Qwen3-TTS custom fields, forwarded via extra_body.
|
# Qwen3-TTS custom fields, forwarded via extra_body.
|
||||||
extra_body: dict[str, Any] = {}
|
extra_body: dict[str, Any] = {}
|
||||||
# Keep this list aligned with vllm_omni.entrypoints.openai.protocol.audio params.
|
# Keep this list aligned with vllm_omni.entrypoints.openai.protocol.audio params.
|
||||||
for key in ("task_type", "ref_text", "ref_audio", "language", "max_new_tokens", "seed"):
|
for key in (
|
||||||
|
"task_type",
|
||||||
|
"ref_text",
|
||||||
|
"ref_audio",
|
||||||
|
"language",
|
||||||
|
"max_new_tokens",
|
||||||
|
"seed",
|
||||||
|
"instructions",
|
||||||
|
"speed",
|
||||||
|
):
|
||||||
if key in request_config:
|
if key in request_config:
|
||||||
extra_body[key] = request_config[key]
|
extra_body[key] = request_config[key]
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user