From 38d440109bb675dac6b8abaece7762d927856d09 Mon Sep 17 00:00:00 2001 From: Jason <72191212+JasonJ2021@users.noreply.github.com> Date: Sat, 2 May 2026 02:39:49 +0800 Subject: [PATCH] [Bugfix] Validate Qwen3-TTS speaker embedding dimensions before engine execution (#3191) Signed-off-by: Jiahui Sun --- .../test_qwen3_tts_speaker_embedding.py | 10 ++--- .../openai_api/test_serving_speech.py | 43 +++++++++++++++++-- .../entrypoints/openai/serving_speech.py | 41 ++++++++++++------ 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/tests/e2e/online_serving/test_qwen3_tts_speaker_embedding.py b/tests/e2e/online_serving/test_qwen3_tts_speaker_embedding.py index b91548c5a..6bff282b5 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_speaker_embedding.py +++ b/tests/e2e/online_serving/test_qwen3_tts_speaker_embedding.py @@ -227,8 +227,8 @@ class TestSpeakerEmbedding1_7B: @pytest.mark.core_model @pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) - def test_1024_dim_on_1_7b_model_rejected_or_errors(self, base_1_7b_server) -> None: - """1024-dim embedding on a 1.7B model (expects 2048) should fail gracefully.""" + def test_1024_dim_on_1_7b_model_rejected(self, base_1_7b_server) -> None: + """1024-dim speaker_embedding on a 1.7B model (expects 2048) should fail gracefully.""" url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech" payload = { "model": MODEL_BASE_1_7B, @@ -243,6 +243,6 @@ class TestSpeakerEmbedding1_7B: response = client.post(url, json=payload) # Wrong dimensions should produce an error, not silent garbage. - assert response.status_code != 200 or len(response.content) < MIN_AUDIO_BYTES, ( - "Expected failure or degenerate output with wrong embedding dimensions" - ) + assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}" + assert "speaker_embedding has 1024 dimensions" in response.text + assert "expected 2048" in response.text diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index b78d62d9e..f94b11dbd 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -853,19 +853,54 @@ class TestTTSMethods: result = speech_server._validate_tts_request(req) assert "non-empty" in result - def test_speaker_embedding_wrong_dims_accepted(self, speech_server): - """Non-standard dimensions pass validation (warning only, not an error).""" - emb = [0.1] * 512 # not 1024 or 2048 + def test_speaker_embedding_wrong_dims_rejected(self, speech_server): + """speaker_embedding dimensions must match the loaded Qwen3-TTS model.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + talker_config=SimpleNamespace(hidden_size=2048), + ) + ) + + emb = [0.1] * 1024 req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True) result = speech_server._validate_tts_request(req) - assert result is None + assert "speaker_embedding has 1024 dimensions" in result + assert "expected 2048" in result def test_speaker_embedding_2048_dims_accepted(self, speech_server): """2048-dim embedding (1.7B model) is accepted without warning.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + talker_config=SimpleNamespace(hidden_size=2048), + ) + ) + emb = [0.1] * 2048 req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True) assert speech_server._validate_tts_request(req) is None + def test_upload_voice_embedding_wrong_dims_rejected(self, speech_server): + """Embedding uploads must match the loaded Qwen3-TTS model before being stored.""" + import json + + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + talker_config=SimpleNamespace(hidden_size=2048), + ) + ) + + with pytest.raises(ValueError, match="expected 2048"): + asyncio.run( + speech_server.upload_voice_embedding( + embedding_json=json.dumps([0.0] * 1024), + consent="consent", + name="bad_emb_voice", + ) + ) + def test_base_task_requires_ref_audio_or_speaker_embedding(self, speech_server): """Base task without ref_audio or speaker_embedding is rejected.""" req = OpenAICreateSpeechRequest(input="Hello", task_type="Base") diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 142d58275..8be9b0976 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -253,6 +253,27 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor) self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor) + def _get_qwen_tts_expected_speaker_embedding_dim(self) -> int | None: + """Return the loaded Qwen3-TTS speaker embedding dim, if known. + + The user-provided speaker embedding is concatenated directly with + talker codec embeddings, so the real compatibility requirement is the + talker hidden size. + """ + if self._tts_model_type != "qwen3_tts": + return None + hf_config = self.engine_client.model_config.hf_config + talker_config = hf_config.talker_config + return int(talker_config.hidden_size) + + def _validate_qwen_tts_speaker_embedding_dim(self, emb_dim: int) -> str | None: + expected_dim = self._get_qwen_tts_expected_speaker_embedding_dim() + if expected_dim is None: + return None + if emb_dim != expected_dim: + return f"speaker_embedding has {emb_dim} dimensions; expected {expected_dim} for the loaded Qwen3-TTS model" + return None + def _load_codec_frame_rate(self) -> float | None: """Load codec frame rate from speech tokenizer config for prompt length estimation.""" try: @@ -757,11 +778,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): raise ValueError("'speaker_embedding' values must be finite (no NaN or Inf)") emb_dim = len(embedding) - if emb_dim not in {1024, 2048}: - logger.warning( - "speaker_embedding has %d dimensions; expected 1024 (0.6B) or 2048 (1.7B)", - emb_dim, - ) + dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_dim) + if dim_err is not None: + raise ValueError(dim_err) voice_name_lower = name.lower() if voice_name_lower in self.uploaded_speakers: @@ -1032,15 +1051,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Base task validation so callers don't need to pass it explicitly. request.x_vector_only_mode = True emb_len = len(request.speaker_embedding) - # ECAPA-TDNN produces 1024-dim (0.6B) or 2048-dim (1.7B) - expected_dims = {1024, 2048} - if emb_len not in expected_dims: - logger.warning( - "speaker_embedding has %d dimensions; expected 1024 " - "(0.6B model) or 2048 (1.7B model). Wrong dimensions " - "will likely result in errors or degraded quality.", - emb_len, - ) + dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_len) + if dim_err is not None: + return dim_err # Validate Base task requirements if task_type == "Base": if request.voice is None: