From d4e90bbba8bc9aea60a8b42650646cc7764c6700 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Wed, 20 May 2026 00:36:59 -0400 Subject: [PATCH] [Bugfix][TTS] Only populate voice_name for uploaded voices without inline ref_audio (#3523) Signed-off-by: Nick Cao Co-authored-by: Claude --- .../openai_api/test_serving_speech_voxcpm.py | 81 +++++++++++++++++++ .../entrypoints/openai/serving_speech.py | 43 ++++++---- 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py b/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py index 1f7279483..f49c1d9ce 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py +++ b/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py @@ -192,3 +192,84 @@ class TestVoxCPM2Serving: Regression: previously returned 400 Invalid voice 'default'. """ assert "default" in voxcpm2_server.supported_speakers + + def test_voxcpm2_inline_ref_audio_does_not_set_voice_name(self, voxcpm2_server): + """Inline ref_audio must not populate voice_name in additional_information. + + Regression: the speaker cache keys on voice_name, so two requests with + the same voice but different inline ref_audio would collide — the second + request reused stale cached features from the first, causing a prefill + length mismatch and engine crash. + """ + mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} + voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) + + request = OpenAICreateSpeechRequest( + input="test", + voice="default", + ref_audio="data:audio/wav;base64,QUJD", + ) + + asyncio.run(voxcpm2_server._prepare_speech_generation(request)) + + additional = mock_prompt["additional_information"] + assert "voice_name" not in additional, ( + "voice_name must not be set for inline ref_audio — " + "it causes speaker cache collisions across different audio clips" + ) + + def test_voxcpm2_uploaded_voice_sets_voice_name(self, voxcpm2_server): + """Uploaded voice without inline ref_audio must populate voice_name. + + This ensures the speaker cache still works for the intended use case: + a voice uploaded once and referenced by name in subsequent requests. + """ + mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} + voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) + voxcpm2_server.uploaded_speakers["my_voice"] = { + "name": "my_voice", + "file_path": "/tmp/fake.wav", + "created_at": 12345, + } + voxcpm2_server.supported_speakers.add("my_voice") + + request = OpenAICreateSpeechRequest( + input="test", + voice="my_voice", + ) + + asyncio.run(voxcpm2_server._prepare_speech_generation(request)) + + additional = mock_prompt["additional_information"] + assert additional.get("voice_name") == "my_voice" + assert "voice_created_at" in additional + + def test_voxcpm2_uploaded_voice_with_inline_override_does_not_set_voice_name(self, voxcpm2_server): + """Uploaded voice with inline ref_audio override must not populate voice_name. + + Prevents cache poisoning: an attacker could send an uploaded voice name + with crafted ref_audio, caching malicious features under the uploaded + voice's name for subsequent legitimate requests. + """ + mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} + voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) + voxcpm2_server.uploaded_speakers["my_voice"] = { + "name": "my_voice", + "file_path": "/tmp/fake.wav", + "created_at": 12345, + } + voxcpm2_server.supported_speakers.add("my_voice") + + request = OpenAICreateSpeechRequest( + input="test", + voice="my_voice", + ref_audio="data:audio/wav;base64,QUJD", + ) + + asyncio.run(voxcpm2_server._prepare_speech_generation(request)) + + additional = mock_prompt["additional_information"] + assert "voice_name" not in additional, ( + "voice_name must not be set when inline ref_audio overrides an " + "uploaded voice — it would poison the cache for future requests" + ) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index e5bfb1329..70a491a94 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1787,6 +1787,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self, request: OpenAICreateSpeechRequest, ref_audio_data: tuple[list[float], int] | None = None, + *, + has_inline_ref_audio: bool = False, ) -> dict[str, Any]: """Build prompt for Fish Speech S2 Pro. @@ -1838,7 +1840,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Pass voice identity for model-side DAC code caching. if request.voice is not None: voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers: + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: additional_information["voice_name"] = voice_lower additional_information["voice_created_at"] = self._voice_created_at(voice_lower) if request.max_new_tokens is not None: @@ -1852,6 +1854,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): async def _build_cosyvoice3_prompt( self, request: OpenAICreateSpeechRequest, + *, + has_inline_ref_audio: bool = False, ) -> dict[str, Any]: """Build prompt for CosyVoice3. @@ -1870,8 +1874,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Pass voice metadata for caching in the processor if request.voice: voice_lower = request.voice.lower() - mm_kwargs["voice_name"] = voice_lower - mm_kwargs["voice_created_at"] = self._voice_created_at(voice_lower) + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: + mm_kwargs["voice_name"] = voice_lower + mm_kwargs["voice_created_at"] = self._voice_created_at(voice_lower) return { "prompt": request.input, @@ -2020,6 +2025,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Resolve uploaded voice for non-Qwen3 models. # Qwen3 TTS has its own uploaded voice handling in _build_tts_params(). + has_inline_ref_audio = request.ref_audio is not None if self._tts_model_type in ("fish_tts", "cosyvoice3", "moss_tts_nano"): err = self._apply_uploaded_speaker(request) if err: @@ -2033,7 +2039,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if request.ref_audio is not None: wav_list, sr = await self._resolve_ref_audio(request.ref_audio) ref_audio_data = (wav_list, sr) - prompt = await self._build_fish_speech_prompt_async(request, ref_audio_data=ref_audio_data) + prompt = await self._build_fish_speech_prompt_async( + request, ref_audio_data=ref_audio_data, has_inline_ref_audio=has_inline_ref_audio + ) tts_params = {} elif self._tts_model_type == "omnivoice": if not request.input or not request.input.strip(): @@ -2050,8 +2058,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt["ref_text"] = request.ref_text if request.voice: voice_lower = request.voice.lower() - prompt["voice_name"] = voice_lower - prompt["voice_created_at"] = self._voice_created_at(voice_lower) + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: + prompt["voice_name"] = voice_lower + prompt["voice_created_at"] = self._voice_created_at(voice_lower) if request.language: prompt["lang"] = request.language if request.instructions: @@ -2068,7 +2077,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if voice_lower not in self.uploaded_speakers and voice_lower not in self.supported_speakers: all_voices = sorted(self.uploaded_speakers.keys() | self.supported_speakers) raise ValueError(f"Invalid voice '{request.voice}'. Supported: {', '.join(all_voices) or 'none'}") - if voice_lower in self.uploaded_speakers: + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: if self.uploaded_speakers[voice_lower].get("embedding_source") == "direct": raise ValueError( f"Uploaded voice '{request.voice}' uses a speaker embedding (Qwen3-only). " @@ -2080,9 +2089,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): tts_params = {} if request.voice: voice_lower = request.voice.lower() - additional = prompt.setdefault("additional_information", {}) - additional["voice_name"] = voice_lower - additional["voice_created_at"] = self._voice_created_at(voice_lower) + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: + additional = prompt.setdefault("additional_information", {}) + additional["voice_name"] = voice_lower + additional["voice_created_at"] = self._voice_created_at(voice_lower) elif self._is_tts: validation_error = self._validate_tts_request(request) if validation_error: @@ -2092,7 +2102,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt = await self._build_voxtral_prompt_async(request) tts_params = {} elif self._tts_model_type == "cosyvoice3": - prompt = await self._build_cosyvoice3_prompt(request) + prompt = await self._build_cosyvoice3_prompt(request, has_inline_ref_audio=has_inline_ref_audio) tts_params = {} elif self._tts_model_type == "ming_flash_omni_tts": prompt = self._build_ming_prompt(request) @@ -2101,8 +2111,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): tts_params = await self._build_moss_tts_params(request) if request.voice: voice_lower = request.voice.lower() - tts_params["voice_name"] = [voice_lower] - tts_params["voice_created_at"] = [self._voice_created_at(voice_lower)] + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: + tts_params["voice_name"] = [voice_lower] + tts_params["voice_created_at"] = [self._voice_created_at(voice_lower)] # Propagate seed from sampling params for deterministic generation. # MOSS-TTS-Nano uses its own internal sampling inside # inference_stream(), which reads seed from additional_information @@ -2375,6 +2386,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): all_voices = sorted(self.uploaded_speakers.keys() | self.supported_speakers) raise ValueError(f"Invalid voice '{request.voice}'. Supported: {', '.join(all_voices) or 'none'}") + has_inline_ref_audio = request.ref_audio is not None err = self._apply_uploaded_speaker(request) if err: raise ValueError(err) @@ -2388,8 +2400,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt["ref_text"] = request.ref_text if request.voice: voice_lower = request.voice.lower() - prompt["voice_name"] = voice_lower - prompt["voice_created_at"] = self._voice_created_at(voice_lower) + if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: + prompt["voice_name"] = voice_lower + prompt["voice_created_at"] = self._voice_created_at(voice_lower) if request.language: prompt["lang"] = request.language if request.instructions: