From 28c87cde19fffacc1a683de93e86812608ef2d3d Mon Sep 17 00:00:00 2001 From: Khairul Kabir <250204754+khairulkabir1661@users.noreply.github.com> Date: Thu, 24 Sep 2026 09:10:01 -0700 Subject: [PATCH] [Bugfix] Replace hardcoded Vivian speaker default with config-ordered default (#5814) Signed-off-by: khairulkabir1661 Co-authored-by: khairulkabir1661 Co-authored-by: Claude Opus 4.6 Co-authored-by: Alex Brooks --- .../openai_api/test_serving_speech.py | 2 +- .../openai_api/test_tts_adapter.py | 112 +++++++++++++++++- .../entrypoints/openai/tts_adapters/base.py | 15 ++- .../openai/tts_adapters/capabilities.py | 13 +- .../openai/tts_adapters/glm_tts.py | 4 +- .../tts_adapters/ming_flash_omni_tts.py | 4 +- .../openai/tts_adapters/ming_tts.py | 4 +- .../openai/tts_adapters/qwen3_tts.py | 17 ++- .../openai/tts_adapters/voxcpm2.py | 4 +- .../openai/tts_adapters/voxtral.py | 2 +- 10 files changed, 157 insertions(+), 20 deletions(-) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 53edd9dc0..46291a88a 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -2535,7 +2535,7 @@ class TestTTSMethods: ) assert speech_server._is_tts is False - assert load_supported_speakers(speech_server.engine_client) == set() + assert load_supported_speakers(speech_server.engine_client) == [] warning.assert_not_called() def test_load_supported_languages_from_config(self, speech_server): diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 22aab89ac..ad89f4954 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -22,7 +22,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( detect_tts_model_type, resolve_adapter, ) -from vllm_omni.entrypoints.openai.tts_adapters.base import resolve_stage_model_path +from vllm_omni.entrypoints.openai.tts_adapters.base import TTSCapabilities, resolve_stage_model_path from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import ( @@ -372,6 +372,10 @@ def test_qwen3_tts_build_constructs_prepared_request(): server = SimpleNamespace(_tts_executor=None, _tts_tokenizer=None, uploaded_speakers={}) engine_client = SimpleNamespace(model_config=SimpleNamespace()) adapter = Qwen3TTSAdapter(SimpleNamespace(server=server, engine_client=engine_client)) + adapter.capabilities = TTSCapabilities( + supported_speakers=frozenset({"vivian", "alice"}), + default_speaker="vivian", + ) async def estimate_prompt_len(_tts_params): return 3 @@ -384,7 +388,7 @@ def test_qwen3_tts_build_constructs_prepared_request(): assert prepared.prompt["prompt_token_ids"] == [1, 1, 1] assert prepared.prompt["additional_information"] is prepared.tts_params assert prepared.tts_params["text"] == ["hello"] - assert prepared.tts_params["speaker"] == ["Vivian"] + assert prepared.tts_params["speaker"] == ["vivian"] assert prepared.model_type == "CustomVoice" @@ -670,5 +674,109 @@ def test_higgs_audio_v2_validate_accepts_plain_text_and_paired_clone() -> None: ) +@pytest.mark.parametrize( + "spk_id_config,expected_default,expected_supported", + [ + ({"charlie": 3, "alice": 1, "bob": 2}, "charlie", frozenset({"alice", "bob", "charlie"})), + ({"vivian": 1}, "vivian", frozenset({"vivian"})), + ({}, None, frozenset()), + (None, None, frozenset()), + ], +) +def test_default_speaker_is_first_in_config(spk_id_config, expected_default, expected_supported): + """Default speaker is the first from config order, not alphabetical (PR #5814).""" + from types import SimpleNamespace + from unittest.mock import Mock + + from vllm_omni.entrypoints.openai.tts_adapters.base import TTSModelAdapter + + mock_ctx = Mock() + mock_ctx.engine_client.model_config.hf_config.talker_config = ( + SimpleNamespace(spk_id=spk_id_config) if spk_id_config is not None else None + ) + + class TestAdapter(TTSModelAdapter): + MODEL_TYPE = "test" + + def validate(self, request): + return None + + async def build(self, request, server): + pass + + caps = TestAdapter(mock_ctx).load_capabilities() + assert caps.default_speaker == expected_default + assert caps.supported_speakers == expected_supported + + +@pytest.mark.parametrize( + "default_speaker,expected_speaker", + [ + ("alice", ["alice"]), + (None, None), + ], +) +def test_qwen3_custom_voice_uses_default_speaker(default_speaker, expected_speaker): + """Qwen3-TTS CustomVoice uses default_speaker, not hardcoded Vivian (PR #5814).""" + from unittest.mock import Mock + + from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext, TTSCapabilities + from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import Qwen3TTSAdapter + + mock_server = Mock() + mock_server.uploaded_speakers = {} + ctx = SpeechServingContext(server=mock_server) + adapter = Qwen3TTSAdapter(ctx) + adapter.capabilities = TTSCapabilities( + supported_speakers=frozenset({"alice", "bob"}) if default_speaker else frozenset(), + default_speaker=default_speaker, + ) + + request = SimpleNamespace( + input="Hello world", + task_type=None, + language=None, + voice=None, + instructions=None, + ref_text=None, + ref_audio=None, + speaker_embedding=None, + x_vector_only_mode=None, + sample_rate=None, + max_new_tokens=None, + initial_codec_chunk_frames=None, + non_streaming_mode=None, + ) + params = adapter._build_tts_params(request) + assert params.get("speaker") == expected_speaker + + +def test_qwen3_validate_rejects_no_voice_no_default_then_accepts(): + """Omitted voice with no default_speaker is rejected; setting a default accepts.""" + from unittest.mock import Mock + + mock_server = Mock() + mock_server.uploaded_speakers = {"uploaded_voice": {"ref_text": "hi"}} + mock_server._get_available_speakers = Mock(return_value={"uploaded_voice"}) + ctx = SpeechServingContext(server=mock_server) + adapter = Qwen3TTSAdapter(ctx) + adapter.capabilities = TTSCapabilities( + supported_speakers=frozenset(), + default_speaker=None, + ) + + request = OpenAICreateSpeechRequest(input="hello") + err = adapter.validate(request) + assert err is not None + assert "voice" in err.lower() + + adapter.capabilities = TTSCapabilities( + supported_speakers=frozenset({"uploaded_voice"}), + default_speaker="uploaded_voice", + ) + request2 = OpenAICreateSpeechRequest(input="hello") + assert adapter.validate(request2) is None + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index cb760e865..8daaca0ea 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -17,8 +17,12 @@ from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar +from vllm.logger import init_logger + from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_codec_frame_rate, load_supported_speakers +logger = init_logger(__name__) + if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -179,6 +183,7 @@ class TTSCapabilities: supported_speakers: frozenset[str] = frozenset() supported_languages: frozenset[str] = DEFAULT_TTS_LANGUAGES codec_frame_rate: float | None = None + default_speaker: str | None = None class TTSModelAdapter(ABC): @@ -315,19 +320,25 @@ class TTSModelAdapter(ABC): return None def load_capabilities(self) -> TTSCapabilities: + speakers_list = list(self._load_supported_speakers()) + default_speaker = speakers_list[0] if speakers_list else None + if default_speaker: + logger.info("Default speaker for CustomVoice task: %s", default_speaker) self.capabilities = TTSCapabilities( precomputed_speakers=self._load_precomputed_speakers(), - supported_speakers=frozenset(self._load_supported_speakers()), + supported_speakers=frozenset(speakers_list), supported_languages=self._load_supported_languages(), codec_frame_rate=self._load_codec_frame_rate(), + default_speaker=default_speaker, ) return self.capabilities def _load_precomputed_speakers(self) -> dict[str, dict[str, Any]]: return {} - def _load_supported_speakers(self) -> set[str]: + def _load_supported_speakers(self) -> list[str]: # Preserve the legacy default path, which reads talker_config. + # Returns a list to preserve config order. return load_supported_speakers(self.ctx.engine_client) def _load_supported_languages(self) -> frozenset[str]: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/capabilities.py b/vllm_omni/entrypoints/openai/tts_adapters/capabilities.py index 310161d82..b1a65eee1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/capabilities.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/capabilities.py @@ -16,23 +16,26 @@ from vllm_omni.utils.speaker_cache import iter_custom_voice_profiles, load_valid logger = init_logger(__name__) -def load_supported_speakers(engine_client: Any, config: Any = None) -> set[str]: - """Extract supported speaker names from a model configuration.""" +def load_supported_speakers(engine_client: Any, config: Any = None) -> list[str]: + """Extract supported speaker names from a model configuration. + + Returns a list to preserve the order from the config file. + """ try: if config is None: config = getattr(engine_client.model_config.hf_config, "talker_config", None) if config is None: - return set() + return [] for attr_name in ("spk_id", "speaker_id"): speakers = config.get(attr_name) if isinstance(config, dict) else getattr(config, attr_name, None) if speakers and isinstance(speakers, dict): - return {speaker.lower() for speaker in speakers} + return [speaker.lower() for speaker in speakers] logger.warning("No speakers found in config (checked spk_id and speaker_id)") except Exception as exc: logger.warning("Could not load speakers from model config: %s", exc) - return set() + return [] def load_codec_frame_rate(engine_client: Any) -> float | None: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index 22dbfc6b6..777d7e17f 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -117,8 +117,8 @@ class GlmTTSAdapter(ARTTSAdapter): prompt = await self._build_prompt(request, has_inline_ref_audio=has_inline_ref_audio) return PreparedRequest(prompt=prompt, tts_params={}, model_type="glm_tts") - def _load_supported_speakers(self) -> set[str]: - return set() + def _load_supported_speakers(self) -> list[str]: + return [] def apply_sampling_overrides( self, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py index 05cd3a64e..5c8d66739 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py @@ -96,6 +96,6 @@ class MingFlashOmniTTSAdapter(ARTTSAdapter): prompt["additional_information"] = info return PreparedRequest(prompt=prompt, tts_params={}, model_type="ming_flash_omni_tts") - def _load_supported_speakers(self) -> set[str]: + def _load_supported_speakers(self) -> list[str]: # Speaker selection is driven by caption JSON rather than a static table. - return set() + return [] diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index 8dc5731a0..12ec3e610 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -354,8 +354,8 @@ class MingTTSAdapter(ARTTSAdapter): except Exception as e: logger.warning(f"Failed to derive Ming codec frame rate from hf_config: {e}") - def _load_supported_speakers(self) -> set[str]: - return set() + def _load_supported_speakers(self) -> list[str]: + return [] def _load_codec_frame_rate(self) -> float | None: codec_frame_rate = self._load_ming_tts_codec_frame_rate() diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index 3a9a0a25d..49940381a 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -165,6 +165,11 @@ class Qwen3TTSAdapter(ARTTSAdapter): ) if request.voice is not None and request.voice not in available_speakers: return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(available_speakers))}" + if request.voice is None and self.capabilities.default_speaker is None: + return ( + "CustomVoice requires a 'voice' parameter when no default speaker " + "is configured. Supported: " + ", ".join(sorted(available_speakers)) + ) # Validate speaker_embedding constraints if request.speaker_embedding is not None: @@ -332,7 +337,17 @@ class Qwen3TTSAdapter(ARTTSAdapter): logger.info("Using precomputed Qwen3-TTS custom voice profile: %s (mode=%s)", voice_lower, mode) elif params["task_type"][0] == "CustomVoice": - params["speaker"] = ["Vivian"] # Default for CustomVoice + if self.capabilities.default_speaker is not None: + params["speaker"] = [self.capabilities.default_speaker] + else: + available = sorted(self.capabilities.supported_speakers) if self.capabilities.supported_speakers else [] + logger.warning( + "CustomVoice task requires a speaker but no default speaker is " + "configured in the model's spk_id config. Available speakers: %s. " + "Request will fail downstream with error from model.", + ", ".join(available) if available else "none", + ) + params["speaker"] = None # Instructions for style/emotion control if request.instructions is not None: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index f4574004e..e32064d34 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -174,8 +174,8 @@ class VoxCPM2Adapter(ARTTSAdapter): validate_profile=validate_voxcpm2_profile, ) - def _load_supported_speakers(self) -> set[str]: - return {"default"} + def _load_supported_speakers(self) -> list[str]: + return ["default"] def apply_sampling_overrides( self, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index 45e41c79c..c0d802d4b 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -108,7 +108,7 @@ class VoxtralTTSAdapter(ARTTSAdapter): prompt = await self._build_prompt_async(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="voxtral_tts") - def _load_supported_speakers(self) -> set[str]: + def _load_supported_speakers(self) -> list[str]: config = self.ctx.engine_client.model_config.hf_config.audio_config return load_supported_speakers(self.ctx.engine_client, config)