diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 61de36938..7fdac15df 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -116,7 +116,7 @@ Content-Type: application/json | `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. | | `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) | -**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian +**Supported languages:** Only applicable to Qwen3-TTS. Derived from the model configuration (`talker_config.codec_language_id` in the checkpoint's `config.json`), plus `Auto`, which is always accepted. Official Qwen3-TTS checkpoints support: Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian. #### Voice Clone Parameters (Base task) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 4fcf3418a..a1fd3392a 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -33,6 +33,7 @@ from vllm_omni.entrypoints.openai.protocol.audio import ( SpeechBatchItem, ) from vllm_omni.entrypoints.openai.serving_speech import ( + _TTS_LANGUAGES, OmniOpenAIServingSpeech, _create_wav_header, ) @@ -1257,6 +1258,67 @@ class TestTTSMethods: # Verify speakers are normalized to lowercase assert server.supported_speakers == {"ryan", "vivian", "aiden"} + def test_load_supported_languages_from_config(self, speech_server): + """Languages/dialects from codec_language_id are loaded title-cased; 'Auto' is added.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + talker_config=SimpleNamespace( + codec_language_id={"chinese": 2055, "english": 2050, "beijing_dialect": 2074} + ) + ) + ) + assert speech_server._load_supported_languages() == { + "Chinese", + "English", + "Beijing_Dialect", + "Auto", + } + + def test_load_supported_languages_from_dict_config(self, speech_server): + """talker_config provided as a plain dict is handled the same as an object.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace(talker_config={"codec_language_id": {"chinese": 2055, "english": 2050}}) + ) + assert speech_server._load_supported_languages() == {"Chinese", "English", "Auto"} + + def test_validate_language_custom_dialect_accepted(self, speech_server): + """A language present in the model config passes validation, case-insensitively.""" + speech_server.supported_languages = {"Chinese", "English", "Beijing_Dialect", "Auto"} + for language in ("Beijing_Dialect", "beijing_dialect", "English", "english", "Auto", "AUTO"): + req = OpenAICreateSpeechRequest(input="Hello", language=language) + result = speech_server._validate_tts_request(req) + assert result is None or "Invalid language" not in result + # Language is normalized to the title-cased config form. + assert req.language == language.title() + + def test_validate_language_unknown_rejected(self, speech_server): + """A language not in the configured set is rejected.""" + speech_server.supported_languages = {"Chinese", "English", "Auto"} + for language in ("Klingon", "klingon"): + req = OpenAICreateSpeechRequest(input="Hello", language=language) + assert "Invalid language" in speech_server._validate_tts_request(req) + + def test_load_supported_languages_default_when_no_config(self, speech_server): + """Empty/missing codec_language_id on a Qwen3-TTS model falls back to the default list.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client.model_config = SimpleNamespace( + hf_config=SimpleNamespace(talker_config=SimpleNamespace(codec_language_id={})) + ) + assert speech_server._load_supported_languages() == _TTS_LANGUAGES + + def test_load_supported_languages_default_on_config_error(self, speech_server): + """If the model config cannot be read, fall back to the default list.""" + speech_server._tts_model_type = "qwen3_tts" + speech_server.engine_client = SimpleNamespace() # no model_config -> AttributeError + assert speech_server._load_supported_languages() == _TTS_LANGUAGES + + def test_load_supported_languages_default_for_non_qwen(self, speech_server): + """Non-qwen3_tts model types get the default language set.""" + speech_server._tts_model_type = None + assert speech_server._load_supported_languages() == _TTS_LANGUAGES + def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture): """Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only).""" speech_server.uploaded_speakers = { diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index bec890cab..1857c6fc8 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -9,6 +9,7 @@ import re import struct import time from collections import OrderedDict +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from http import HTTPStatus from pathlib import Path @@ -102,19 +103,21 @@ _SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES = { "higgs_audio_v2", "higgs_audio_v3", } -_TTS_LANGUAGES: set[str] = { - "Auto", - "Chinese", - "English", - "Japanese", - "Korean", - "German", - "French", - "Russian", - "Portuguese", - "Spanish", - "Italian", -} +_TTS_LANGUAGES = frozenset( + { + "Auto", + "Chinese", + "English", + "Japanese", + "Korean", + "German", + "French", + "Russian", + "Portuguese", + "Spanish", + "Italian", + } +) _REF_AUDIO_MIN_DURATION = 1.0 # seconds _REF_AUDIO_MAX_DURATION = 30.0 # seconds _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256 @@ -434,6 +437,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Merge built-in speakers into the set initialized by _init_speaker_storage. self.supported_speakers |= self._load_supported_speakers() self.supported_speakers |= set(self.precomputed_speakers) + + self.supported_languages = self._load_supported_languages() + self._tts_tokenizer = None self._voxcpm2_tokenizer = None self._voxcpm2_split_map: dict[int, list[int]] = {} @@ -694,6 +700,26 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return set() + def _load_supported_languages(self) -> frozenset[str]: + """Load supported languages (title-cased) from the model configuration""" + if self._tts_model_type != "qwen3_tts": + return _TTS_LANGUAGES + try: + config = self.engine_client.model_config.hf_config.talker_config + + if isinstance(config, dict): + codec_language_id = config.get("codec_language_id") + else: + codec_language_id = getattr(config, "codec_language_id", None) + + if codec_language_id and isinstance(codec_language_id, Mapping): + return frozenset(str(language).title() for language in codec_language_id) | {"Auto"} + + logger.warning("No codec_language_id found in talker_config; falling back to default languages") + except Exception as e: + logger.warning("Could not load languages from model config: %s", e) + return _TTS_LANGUAGES + def _estimate_ref_code_len(self, ref_audio: object) -> int | None: """Estimate ref_code length from ref_audio waveform without running the codec. @@ -1420,9 +1446,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if not request.input or not request.input.strip(): return "Input text cannot be empty" - # Validate language - if request.language is not None and request.language not in _TTS_LANGUAGES: - return f"Invalid language '{request.language}'. Supported: {', '.join(sorted(_TTS_LANGUAGES))}" + # Validate language (case-insensitive; normalized to the title-cased config form) + if request.language is not None: + request.language = request.language.title() + if request.language not in self.supported_languages: + return ( + f"Invalid language '{request.language}'. Supported: {', '.join(sorted(self.supported_languages))}" + ) # Validate speaker for CustomVoice task if task_type == "CustomVoice":