From feefdaee9fdbb62ce3fdbeb2814e20d70a245573 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:07:01 +0200 Subject: [PATCH] [Bugfix] Accept 'speaker' as alias for 'voice' in TTS speech API (#2424) Signed-off-by: marksverdhei Signed-off-by: marksverdhai <249650165+marksverdhai@users.noreply.github.com> Co-authored-by: marksverdhai <249650165+marksverdhai@users.noreply.github.com> --- .../qwen3_tts/openai_speech_client.py | 2 +- .../openai_api/test_serving_speech.py | 175 ++++++++++++++++++ .../entrypoints/openai/protocol/audio.py | 6 +- .../entrypoints/openai/serving_speech.py | 88 +++++++-- 4 files changed, 255 insertions(+), 16 deletions(-) diff --git a/examples/online_serving/qwen3_tts/openai_speech_client.py b/examples/online_serving/qwen3_tts/openai_speech_client.py index 4741a4715..77e13b08e 100644 --- a/examples/online_serving/qwen3_tts/openai_speech_client.py +++ b/examples/online_serving/qwen3_tts/openai_speech_client.py @@ -71,7 +71,7 @@ def run_tts_generation(args) -> None: payload = { "model": args.model, "input": args.text, - "speaker": args.speaker, + "voice": args.speaker, "response_format": args.response_format, } diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index da15ec8f0..334264602 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -1028,6 +1028,181 @@ class TestTTSMethods: assert result is None + # ── speaker field alias ── + + def test_speaker_alias_accepted_as_voice(self): + """The 'speaker' JSON key should be accepted as an alias for 'voice'.""" + req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "custom_voice"}) + assert req.voice == "custom_voice" + + def test_voice_field_still_accepted(self): + """The canonical 'voice' JSON key should still work.""" + req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "voice": "custom_voice"}) + assert req.voice == "custom_voice" + + def test_speaker_alias_in_base_task_with_uploaded_voice(self, speech_server): + """Using 'speaker' key with an uploaded voice should work for Base task.""" + speech_server.uploaded_speakers = { + "utesf": { + "name": "UTESF", + "file_path": "/tmp/voice_samples/utesf.wav", + "mime_type": "audio/wav", + "ref_text": None, + } + } + req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"}) + assert req.voice == "UTESF" + with patch("pathlib.Path.exists", return_value=True): + result = speech_server._validate_qwen_tts_request(req) + assert result is None + + # ── uploaded voice with embedding ── + + def test_build_tts_params_with_uploaded_voice_embedding(self, speech_server): + """Test _build_tts_params loads embedding for embedding-uploaded voices.""" + speech_server.uploaded_speakers = { + "emb_voice": { + "name": "emb_voice", + "file_path": "/tmp/voice_samples/emb_voice.safetensors", + "mime_type": "application/x-safetensors", + "embedding_source": "direct", + "embedding_dim": 1024, + "cache_status": "ready", + "cache_file": "/tmp/voice_samples/emb_voice.safetensors", + } + } + speech_server.supported_speakers = {"ryan", "vivian", "emb_voice"} + + fake_embedding = [0.1] * 1024 + with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_get_emb: + mock_get_emb.return_value = fake_embedding + req = OpenAICreateSpeechRequest(input="Hello", voice="emb_voice") + params = speech_server._build_tts_params(req) + + assert "voice_clone_prompt" in params + assert params["voice_clone_prompt"][0]["ref_spk_embedding"] == fake_embedding + assert params["task_type"] == ["Base"] + assert params["x_vector_only_mode"] == [True] + assert "ref_audio" not in params + + # ── regression: full flow from issue #1603 ── + + def test_regression_1603_speaker_key_with_uploaded_audio_voice(self, speech_server): + """Regression test for #1603: upload audio voice, then invoke TTS with 'speaker' key. + + Verifies the full validate → build_params pipeline works end-to-end. + """ + speech_server.uploaded_speakers = { + "utesf": { + "name": "UTESF", + "file_path": "/tmp/voice_samples/utesf.wav", + "mime_type": "audio/wav", + "ref_text": "Hola, esta es una prueba.", + } + } + # Parse with 'speaker' alias (the key users actually send) + req = OpenAICreateSpeechRequest.model_validate( + {"input": "Hello world", "speaker": "UTESF", "task_type": "Base"} + ) + assert req.voice == "UTESF" + + # Validation should pass (file exists) + with patch("pathlib.Path.exists", return_value=True): + err = speech_server._validate_qwen_tts_request(req) + assert err is None, f"Validation failed: {err}" + + # Build params should auto-set ref_audio from stored file + with patch.object(speech_server, "_get_uploaded_audio_data") as mock_audio: + mock_audio.return_value = "data:audio/wav;base64,ZmFrZQ==" + params = speech_server._build_tts_params(req) + + assert params["task_type"] == ["Base"] + assert params["ref_audio"] == ["data:audio/wav;base64,ZmFrZQ=="] + assert params["ref_text"] == ["Hola, esta es una prueba."] + assert params["x_vector_only_mode"] == [False] + assert params["speaker"] == ["utesf"] + + def test_regression_1603_speaker_key_with_uploaded_embedding_voice(self, speech_server): + """Regression test for #1603: upload embedding voice, then invoke TTS with 'speaker' key. + + Verifies embedding-uploaded voices are loaded as voice_clone_prompt, not as audio. + """ + speech_server.uploaded_speakers = { + "myvoice": { + "name": "myvoice", + "file_path": "/tmp/voice_samples/myvoice.safetensors", + "mime_type": "application/x-safetensors", + "embedding_source": "direct", + "embedding_dim": 1024, + "cache_status": "ready", + "cache_file": "/tmp/voice_samples/myvoice.safetensors", + } + } + # Parse with 'speaker' alias + req = OpenAICreateSpeechRequest.model_validate( + {"input": "Hello world", "speaker": "myvoice", "task_type": "Base"} + ) + assert req.voice == "myvoice" + + # Validation should pass + with patch("pathlib.Path.exists", return_value=True): + err = speech_server._validate_qwen_tts_request(req) + assert err is None, f"Validation failed: {err}" + + # Build params should use embedding, NOT audio + fake_emb = [0.1] * 1024 + with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_emb: + mock_emb.return_value = fake_emb + params = speech_server._build_tts_params(req) + + assert params["task_type"] == ["Base"] + assert params["x_vector_only_mode"] == [True] + assert "voice_clone_prompt" in params + assert params["voice_clone_prompt"][0]["ref_spk_embedding"] == fake_emb + # Must NOT have ref_audio — that would fail for safetensors files + assert "ref_audio" not in params + + def test_validate_rejects_embedding_voice_with_pending_cache(self, speech_server): + """Validation should reject embedding voices whose cache is not yet ready.""" + speech_server.uploaded_speakers = { + "myvoice": { + "name": "myvoice", + "file_path": "/tmp/myvoice.safetensors", + "mime_type": "application/x-safetensors", + "embedding_source": "direct", + "cache_status": "pending", + "cache_file": None, + } + } + req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "myvoice", "task_type": "Base"}) + with patch("pathlib.Path.exists", return_value=True): + err = speech_server._validate_qwen_tts_request(req) + assert err is not None + assert "not yet ready" in err + + def test_x_vector_only_mode_not_overwritten_for_uploaded_embedding(self, speech_server): + """x_vector_only_mode set by uploaded embedding must not be overwritten by request field.""" + speech_server.uploaded_speakers = { + "emb_voice": { + "name": "emb_voice", + "file_path": "/tmp/emb_voice.safetensors", + "mime_type": "application/x-safetensors", + "embedding_source": "direct", + "embedding_dim": 1024, + "cache_status": "ready", + "cache_file": "/tmp/emb_voice.safetensors", + } + } + fake_emb = [0.1] * 1024 + with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_emb: + mock_emb.return_value = fake_emb + # Client explicitly sends x_vector_only_mode=False, but embedding requires True + req = OpenAICreateSpeechRequest(input="Hello", voice="emb_voice", x_vector_only_mode=False) + params = speech_server._build_tts_params(req) + + assert params["x_vector_only_mode"] == [True] + assert "voice_clone_prompt" in params + def test_max_instructions_length_default(self, speech_server): """Test default max instructions length (500) when no config provided.""" # Fixture creates server with no CLI override and no TTS stage diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 89d2dc02f..8468efd86 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -2,7 +2,7 @@ import math from typing import Literal import numpy as np -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator _MAX_EMBEDDING_DIM = 8192 @@ -10,8 +10,12 @@ _MAX_EMBEDDING_DIM = 8192 class OpenAICreateSpeechRequest(BaseModel): input: str model: str | None = None + # Accept both "voice" (OpenAI convention) and "speaker" (model/internal + # convention) as input keys. Intentionally global — all TTS backends + # (Qwen3-TTS, Voxtral, Fish Speech) use this field for the speaker name. voice: str | None = Field( default=None, + validation_alias=AliasChoices("voice", "speaker"), description="Speaker/voice to use. For Qwen3-TTS: vivian, ryan, aiden, etc.", ) instructions: str | None = Field( diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 10c5fdacc..a4b029393 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -467,6 +467,48 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): logger.error(f"Could not read audio file for voice {voice_name}: {e}") return None + def _get_uploaded_speaker_embedding(self, voice_name: str) -> list[float] | None: + """Load pre-computed speaker embedding for an uploaded voice. + + Returns the embedding as a list of floats, or None if the voice + was not uploaded with an embedding (i.e. it has audio instead). + """ + voice_name_lower = voice_name.lower() + if voice_name_lower not in self.uploaded_speakers: + return None + + speaker_info = self.uploaded_speakers[voice_name_lower] + if speaker_info.get("embedding_source") != "direct": + return None + + cache_file = speaker_info.get("cache_file") + if not cache_file or not Path(cache_file).exists(): + logger.warning("Embedding file not found for voice %s: %s", voice_name, cache_file) + return None + + if not _validate_path_within_directory(Path(cache_file), self.uploaded_speakers_dir): + logger.error("Cache file path traversal detected for voice %s: %s", voice_name, cache_file) + return None + + try: + from safetensors.torch import load_file + except ImportError: + logger.error( + "The 'safetensors' package is required to load speaker embeddings. " + "Install it with: pip install safetensors" + ) + return None + + try: + tensors = load_file(cache_file) + if "speaker_embedding" not in tensors: + logger.warning("Key 'speaker_embedding' not found in %s for voice %s", cache_file, voice_name) + return None + return tensors["speaker_embedding"].squeeze().tolist() + except Exception as e: + logger.error("Could not load embedding for voice %s: %s", voice_name, e) + return None + async def upload_voice( self, audio_file: UploadFile, @@ -858,11 +900,17 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # voice is not None voice_lower = request.voice.lower() if voice_lower in self.uploaded_speakers: - # Check if audio file exists for uploaded speaker + # Check if data file exists for uploaded speaker speaker_info = self.uploaded_speakers[voice_lower] file_path = Path(speaker_info["file_path"]) if not file_path.exists(): - return f"Audio file for uploaded speaker '{request.voice}' not found on disk" + return f"Data file for uploaded speaker '{request.voice}' not found on disk" + # For embedding-uploaded voices, verify the cache is ready + if speaker_info.get("embedding_source") == "direct": + cache_file = speaker_info.get("cache_file") + if not cache_file or not Path(cache_file).exists(): + status = speaker_info.get("cache_status", "unknown") + return f"Speaker embedding for '{request.voice}' is not yet ready (cache_status='{status}')" else: # need ref_audio for built-in speaker if request.ref_audio is None: @@ -1107,20 +1155,32 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Uploaded voices use task_type="Base" (CustomVoice requires built-in spk_id). # If ref_text was provided at upload time, use in-context cloning; otherwise x_vector only. if request.voice.lower() in self.uploaded_speakers and request.ref_audio is None: - audio_data = self._get_uploaded_audio_data(request.voice) - if not audio_data: - raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted") speaker_info = self.uploaded_speakers[request.voice.lower()] - stored_ref_text = speaker_info.get("ref_text") - params["ref_audio"] = [audio_data] - params["task_type"] = ["Base"] - params["voice_created_at"] = [speaker_info.get("created_at", 0)] - if stored_ref_text: - params["ref_text"] = [stored_ref_text] - params["x_vector_only_mode"] = [False] + + # Check if this voice was uploaded with a pre-computed embedding. + # Populate request.speaker_embedding so the existing code path + # (below) handles voice_clone_prompt and x_vector_only_mode. + embedding = self._get_uploaded_speaker_embedding(request.voice) + if embedding is not None: + request.speaker_embedding = embedding + params["task_type"] = ["Base"] + logger.info("Auto-set speaker_embedding for uploaded voice: %s", request.voice) else: - params["x_vector_only_mode"] = [True] - logger.info("Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text)) + audio_data = self._get_uploaded_audio_data(request.voice) + if not audio_data: + raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted") + stored_ref_text = speaker_info.get("ref_text") + params["ref_audio"] = [audio_data] + params["task_type"] = ["Base"] + params["voice_created_at"] = [speaker_info.get("created_at", 0)] + if stored_ref_text: + params["ref_text"] = [stored_ref_text] + params["x_vector_only_mode"] = [False] + else: + params["x_vector_only_mode"] = [True] + logger.info( + "Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text) + ) elif params["task_type"][0] == "CustomVoice": params["speaker"] = ["Vivian"] # Default for CustomVoice