diff --git a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py index e7371fe17..e564baf9d 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py +++ b/tests/e2e/online_serving/test_higgs_audio_v2_expansion.py @@ -6,7 +6,7 @@ E2E Online expansion tests for higgs-audio v2 against /v1/audio/speech. v1 scope is plain text -> 24 kHz speech plus shallow voice clone via ref_audio + ref_text (inline) or voice= (after POST /v1/audio/voices). The model-aware request validator -(vllm_omni/entrypoints/openai/serving_speech.py::_validate_higgs_audio_v2_request) +(vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2::validate) rejects multi-speaker tags, language overrides, task_type, and bare voice= for names that do not match an uploaded speaker — this suite exercises both the happy path (plain text in, audio bytes out) and the diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 904e5735c..26a66450c 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -1549,7 +1549,7 @@ class TestTTSMethods: req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"}) assert req.voice == "UTESF" mocker.patch("pathlib.Path.exists", return_value=True) - result = speech_server._validate_qwen_tts_request(req) + result = speech_server._validate_tts_request(req) assert result is None # ── uploaded voice with embedding ── @@ -1684,7 +1684,7 @@ class TestTTSMethods: # Validation should pass (file exists) mocker.patch("pathlib.Path.exists", return_value=True) - err = speech_server._validate_qwen_tts_request(req) + err = speech_server._validate_tts_request(req) assert err is None, f"Validation failed: {err}" # Build params should auto-set ref_audio from stored file @@ -1722,7 +1722,7 @@ class TestTTSMethods: # Validation should pass mocker.patch("pathlib.Path.exists", return_value=True) - err = speech_server._validate_qwen_tts_request(req) + err = speech_server._validate_tts_request(req) assert err is None, f"Validation failed: {err}" # Build params should use embedding, NOT audio @@ -1866,6 +1866,35 @@ class TestTTSMethods: assert error is not None assert "max 10 characters" in error + def test_batch_voice_not_treated_as_inline_ref_audio( + self, + speech_server, + mocker: MockerFixture, + ): + speech_server._check_model = mocker.AsyncMock(return_value=None) + speech_server._batch_max_items = 32 + + def validate_and_resolve_uploaded_voice(request): + assert request.voice == "alice" + assert request.ref_audio is None + + request.ref_audio = "data:audio/wav;base64,dGVzdA==" + request.ref_text = "reference text" + return None + + speech_server._validate_tts_request = mocker.Mock(side_effect=validate_and_resolve_uploaded_voice) + speech_server._generate_audio_bytes = mocker.AsyncMock(return_value=("YWJj", "audio/wav")) + + batch = BatchSpeechRequest(voice="alice", items=[SpeechBatchItem(input="hello")]) + + response = asyncio.run(speech_server.create_speech_batch(batch)) + call = speech_server._generate_audio_bytes.await_args + request = call.args[0] + + assert response.results[0].status == "success" + assert request.ref_audio is not None + assert call.kwargs["has_inline_ref_audio"] is False + class TestFileValidationFunctions: """Unit tests for file validation helper functions.""" @@ -3221,25 +3250,25 @@ class TestCosyVoice3Serving: def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "empty" in error.lower() def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "ref_audio" in error.lower() def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "ref_text" in error.lower() def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello") - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "url" in error.lower() or "format" in error.lower() @@ -3249,7 +3278,7 @@ class TestCosyVoice3Serving: ref_audio="data:audio/wav;base64,abc123", ref_text="Reference transcript", ) - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is None def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server): @@ -3259,7 +3288,7 @@ class TestCosyVoice3Serving: ref_text="hello", max_new_tokens=0, ) - error = cosyvoice3_server._validate_cosyvoice3_request(request) + error = cosyvoice3_server._validate_tts_request(request) assert error is not None assert "max_new_tokens" in error @@ -3325,13 +3354,13 @@ def glm_tts_server(mocker: MockerFixture): class TestGLMTTSServing: def test_validate_glm_tts_requires_ref_audio(self, glm_tts_server): request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript") - error = glm_tts_server._validate_glm_tts_request(request) + error = glm_tts_server._validate_tts_request(request) assert error is not None assert "ref_audio" in error def test_validate_glm_tts_requires_ref_text(self, glm_tts_server): request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") - error = glm_tts_server._validate_glm_tts_request(request) + error = glm_tts_server._validate_tts_request(request) assert error is not None assert "ref_text" in error @@ -3438,7 +3467,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture): """Qwen3 TTS path should call _estimate_prompt_len_async.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3455,7 +3484,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server.engine_client.default_sampling_params_list = [ SimpleNamespace(max_tokens=2048, seed=42, extra_args=None) ] - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3505,7 +3534,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=sse should request delta-style multimodal outputs.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3522,7 +3551,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream=True should request delta-style multimodal outputs for SSE streaming.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3539,7 +3568,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=audio should request delta-style multimodal outputs.""" - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) @@ -3559,7 +3588,7 @@ class TestTTSAsyncOffloading: ): """Full-payload TTS streaming should not request delta multimodal outputs.""" qwen3_tts_server.engine_client.model_config.async_chunk = False - qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) + qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) qwen3_tts_server._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 3db6b63a6..a935157e1 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -150,7 +150,6 @@ _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256 _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024 _QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key" _TTS_MAX_INSTRUCTIONS_LENGTH = 500 -_TTS_MAX_NEW_TOKENS_MIN = 1 _TTS_MAX_NEW_TOKENS_MAX = 4096 _MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT @@ -1490,48 +1489,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate TTS request parameters. Returns error message or None.""" - if self._tts_model_type == "step_audio2": - # StepAudio2 only requires non-empty input text - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - return None - if self._tts_model_type == "voxtral_tts": - return self._validate_voxtral_tts_request(request) - if self._tts_model_type == "fish_tts": - return self._validate_fish_tts_request(request) - if self._tts_model_type == "cosyvoice3": - return self._validate_cosyvoice3_request(request) - if self._tts_model_type == "voxcpm2": - return self._validate_voxcpm2_request(request) if self._tts_model_type == "ming_flash_omni_tts": return self._validate_ming_flash_omni_tts_request(request) - if self._tts_model_type == "ming_tts": - return self._validate_ming_tts_request(request) - if self._tts_model_type in ("moss_tts_nano", "moss_tts"): - return self._validate_moss_tts_request(request) - if self._tts_model_type == "glm_tts": - return self._validate_glm_tts_request(request) - return self._validate_qwen_tts_request(request) - def _validate_voxcpm2_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate VoxCPM2 request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" + adapter = self._get_tts_adapter() + if adapter is not None: + return adapter.validate(request) - if request.voice is not None: - request.voice = request.voice.lower() - available_voices = set(self.uploaded_speakers) | set(self.precomputed_speakers) | {"default"} - if request.voice not in available_voices: - supported = ", ".join(sorted(available_voices)) or "none" - return f"Invalid voice '{request.voice}'. Supported: {supported}" + adapter_cls = resolve_adapter("qwen3_tts") + if adapter_cls is None: + raise ValueError("Qwen3-TTS adapter is not registered") - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None + ctx = SpeechServingContext( + server=self, + engine_client=self.engine_client, + ) + return adapter_cls(ctx).validate(request) def _voxcpm2_encode(self, text: str) -> list[int]: """Tokenize text for VoxCPM2, splitting multichar Chinese tokens.""" @@ -1599,158 +1572,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" return None - def _validate_voxtral_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Voxtral TTS request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # Voxtral TTS requires either a preset voice or ref_audio for voice cloning. - if request.voice is None and request.ref_audio is None: - return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided" - - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if request.voice is not None: - request.voice = request.voice.lower() - if self.supported_speakers and request.voice not in self.supported_speakers: - return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Qwen TTS request parameters. Returns error message or None.""" - # Infer Base task when ref_audio or ref_text is provided without explicit task_type. - if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None): - request.task_type = "Base" - - # Normalize voice to lowercase for case-insensitive matching - if request.voice is not None: - request.voice = request.voice.lower() - if request.task_type is None and request.voice in self.precomputed_speakers: - request.task_type = "Base" - task_type = request.task_type or "CustomVoice" - - # Validate input is not empty - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # Validate language (case-insensitive; normalized to the title-cased config form) - if request.language is not None: - request.language = request.language.title() - if request.language not in self.supported_languages: - return ( - f"Invalid language '{request.language}'. Supported: {', '.join(sorted(self.supported_languages))}" - ) - - # Validate speaker for CustomVoice task - if task_type == "CustomVoice": - if not self.supported_speakers: - return ( - "This model does not support CustomVoice task (no speakers configured). " - "Use task_type='Base' with ref_audio/ref_text for voice cloning, " - "or use a CustomVoice model." - ) - if request.voice is not None and request.voice not in self.supported_speakers: - return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}" - - # Validate speaker_embedding constraints - if request.speaker_embedding is not None: - if task_type != "Base": - return "'speaker_embedding' is only valid for Base task" - if not request.speaker_embedding: - return "'speaker_embedding' must be a non-empty list of floats" - # speaker_embedding implies x_vector_only_mode — set it before - # Base task validation so callers don't need to pass it explicitly. - request.x_vector_only_mode = True - emb_len = len(request.speaker_embedding) - dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_len) - if dim_err is not None: - return dim_err - # Validate Base task requirements - if task_type == "Base": - if request.voice is None: - # 1. Ensure a voice source is provided - if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None: - return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning" - # 2. Validate ref_audio format if it exists (using the helper from main) - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - # 3. Validate text requirements based on the mode - if not getattr(request, "x_vector_only_mode", False): - if not request.ref_text or not request.ref_text.strip(): - return ( - "Base task requires non-empty 'ref_text' (transcript of " - "the reference audio) unless 'x_vector_only_mode' is enabled" - ) - else: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers: - # Check if data file exists for uploaded speaker - speaker_info = self.uploaded_speakers[voice_lower] - file_path = Path(speaker_info["file_path"]) - if not file_path.exists(): - return f"Data file for uploaded speaker '{request.voice}' not found on disk" - elif voice_lower in self.precomputed_speakers: - profile = self.precomputed_speakers[voice_lower] - mode = str(profile.get("mode") or "xvec").lower() - ref_text = request.ref_text or profile.get("ref_text") - if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()): - return ( - f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in " - "the request or custom voice manifest" - ) - else: - # need ref_audio for built-in speaker - if request.ref_audio is None: - return ( - f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning" - ) - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not getattr(request, "x_vector_only_mode", False) and ( - not request.ref_text or not request.ref_text.strip() - ): - return ( - "Base task requires non-empty 'ref_text' (transcript of " - "the reference audio) unless 'x_vector_only_mode' is enabled" - ) - - # Validate cross-parameter dependencies - if task_type != "Base": - if request.ref_text is not None: - return "'ref_text' is only valid for Base task" - if request.x_vector_only_mode is not None: - return "'x_vector_only_mode' is only valid for Base task" - - # Validate VoiceDesign task requirements - if task_type == "VoiceDesign" and not request.instructions: - return "VoiceDesign task requires 'instructions' to describe the voice" - - # Validate instructions length (using cached value from initialization) - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - # Validate max_new_tokens range - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - def _detect_moss_variant(self) -> str: """Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS variant family (tts, ttsd, sound_effect, voice_generator, realtime). @@ -1776,66 +1597,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "voice_generator" return "tts" - def _validate_moss_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate any MOSS-TTS-family request (nano + 5 full variants). - - Dispatches by ``self._moss_variant``: - - ``tts``/``realtime``: require ``ref_audio`` (voice cloning). - - ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2`` - optional (defaults to the same ref for both speakers). - - ``sound_effect``: require ``ambient_sound`` (no ref_audio). - - ``voice_generator``: require ``instructions`` (no ref_audio). - - For the legacy moss_tts_nano model_type the variant is None and - we fall through to the original nano contract (ref_audio only). - """ - if not request.input or not request.input.strip(): - # SoundEffect can legitimately have empty input (just ambient_sound). - if self._moss_variant != "sound_effect": - return "Input text cannot be empty" - - v = self._moss_variant - if v in (None, "tts", "realtime", "local"): - if request.ref_audio is None: - label = ( - "MOSS-TTS-Nano" - if v is None - else ( - "MOSS-TTS-Realtime" - if v == "realtime" - else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS") - ) - ) - return f"{label} requires 'ref_audio' (reference audio for voice cloning)." - return self._validate_ref_audio_format(request.ref_audio) - - if v == "ttsd": - if request.ref_audio is None: - return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)." - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if request.ref_audio_2 is not None: - return self._validate_ref_audio_format(request.ref_audio_2) - return None - - if v == "sound_effect": - if not request.ambient_sound or not request.ambient_sound.strip(): - return ( - "MOSS-SoundEffect requires 'ambient_sound' (natural language " - "description of the sound effect to synthesise)." - ) - return None - - if v == "voice_generator": - if not request.instructions or not request.instructions.strip(): - return ( - "MOSS-VoiceGenerator requires 'instructions' (natural language " - "voice description, e.g. 'a warm female voice with an American accent')." - ) - return None - - return None # unreachable - def _get_moss_processor(self): """Lazily load the upstream MOSS-TTS processor once per server. @@ -1994,93 +1755,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): params["max_new_frames"] = [request.max_new_tokens] return params - def _validate_higgs_audio_v2_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate higgs_audio_v2 request parameters. Returns error message or None. - - Accepted: plain text -> speech, or shallow voice clone via ``ref_audio`` - + ``ref_text`` (both required together). Still out of scope: preset - ``voice``/``speaker`` selection, ``x_vector_only_mode`` / - ``speaker_embedding`` helpers, ``task_type``/``language``/ - ``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]`` - tags inside the input body. - """ - from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( - MULTI_SPEAKER_TAG_PATTERN, - ) - - if not request.input or not request.input.strip(): - return "higgs_audio_v2: input text cannot be empty" - - # Voice clone: ref_audio and ref_text must come together. - if request.ref_audio is not None and not request.ref_text: - return ( - "higgs_audio_v2 voice clone requires both 'ref_audio' and " - "'ref_text'; received ref_audio without ref_text" - ) - if request.ref_text and request.ref_audio is None: - return ( - "higgs_audio_v2 voice clone requires both 'ref_audio' and " - "'ref_text'; received ref_text without ref_audio" - ) - - if request.x_vector_only_mode is not None: - return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)" - if request.speaker_embedding is not None: - return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)" - if request.voice and request.ref_audio is None: - # _apply_uploaded_speaker runs before this validator; if voice was - # an uploaded speaker, ref_audio is now populated and ref_text is - # backfilled from the speaker entry. A bare voice= with no - # ref_audio means the name didn't resolve to an uploaded speaker - # (and higgs has no built-in preset voices). - return ( - "higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; " - f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. " - f"Got voice={request.voice!r}" - ) - if request.instructions: - return ( - "higgs_audio_v2 v1 does not support 'instructions' (voice " - "style/emotion control); supply plain text instead" - ) - if request.task_type is not None: - return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech" - if request.language is not None: - return ( - "higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text" - ) - if request.speed is not None and request.speed != 1.0: - return ( - "higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)" - ) - - if MULTI_SPEAKER_TAG_PATTERN.search(request.input): - return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - - def _validate_glm_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate GLM-TTS request — requires ref_audio for voice cloning.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if request.ref_audio is None: - return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning" - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not request.ref_text or not request.ref_text.strip(): - return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be >= {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - return None - async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest): """Build prompt_token_ids for higgs_audio_v2 via the upstream processor. @@ -2157,18 +1831,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # ---- higgs-audio v3 ---- - def _validate_higgs_audio_v3_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate higgs_audio_v3 request parameters.""" - if not request.input or not request.input.strip(): - return "higgs_audio_v3: input text cannot be empty" - if request.ref_audio is not None and not request.ref_text: - # Voice clone ref_text is optional for v3 (improves fidelity but not required) - pass - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - return None - async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest): """Build prompt_token_ids for higgs_audio_v3. @@ -2302,143 +1964,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._higgs_audio_v3_adapter = adapter return adapter - def _validate_fish_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Fish Speech request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not request.ref_text or not request.ref_text.strip(): - return "Voice cloning requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate CosyVoice3 request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - # CosyVoice3 requires reference audio for voice cloning - if request.ref_audio is None: - return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)" - - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if not request.ref_text or not request.ref_text.strip(): - return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Ming TTS request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if isinstance(request.ref_audio, list): - return self._validate_ming_tts_podcast_request(request) - return self._validate_ming_tts_single_speaker_request(request) - - def _validate_ming_tts_single_speaker_request(self, request: OpenAICreateSpeechRequest) -> str | None: - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - - if request.speaker_embedding is not None: - if not request.speaker_embedding: - return "'speaker_embedding' must be a non-empty list of floats" - emb_len = len(request.speaker_embedding) - if emb_len != 192: - logger.warning( - "speaker_embedding has %d dimensions; Ming dense expects 192. " - "Wrong dimensions will likely fail or degrade output.", - emb_len, - ) - - voice_lower = request.voice.lower() if isinstance(request.voice, str) else None - uploaded_voice = bool(voice_lower and voice_lower in self.uploaded_speakers) - clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice - - if request.task_type == "Base" and not clone_source_present: - return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample" - - if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip(): - return "'ref_text' must be non-empty when provided with 'ref_audio'" - - # Ming offline ref-audio cases use prompt_waveform without prompt_text; - # keep the transcript requirement for other TTS models. - if request.ref_audio is not None and request.speaker_embedding is None and not self._is_ming_tts_model(): - uploaded_ref_text = self.uploaded_speakers[voice_lower].get("ref_text") if uploaded_voice else None - if not (request.ref_text and request.ref_text.strip()) and not uploaded_ref_text: - return "Reference-audio cloning requires non-empty 'ref_text'" - - if request.ref_text is not None and request.ref_audio is None and not uploaded_voice: - return "'ref_text' requires 'ref_audio' or an uploaded voice sample" - - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - - def _validate_ming_tts_podcast_request(self, request: OpenAICreateSpeechRequest) -> str | None: - if len(request.ref_audio) < 2: - return "Podcast-style Ming requests require at least two 'ref_audio' clips" - - for ref_audio in request.ref_audio: - fmt_err = self._validate_ref_audio_format(ref_audio) - if fmt_err: - return fmt_err - - if not request.ref_text or not request.ref_text.strip(): - return "Podcast-style Ming requests require non-empty 'ref_text'" - - if request.speaker_embedding is not None: - embeddings = request.speaker_embedding - embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1 - if embedding_count != len(request.ref_audio): - return ( - "Podcast-style Ming requests require one speaker embedding per ref_audio clip; " - f"got {embedding_count} embeddings for {len(request.ref_audio)} clips" - ) - if embeddings and isinstance(embeddings[0], list): - for item in embeddings: - if len(item) != 192: - return "Podcast-style Ming speaker embeddings must each have 192 dimensions" - - if request.instructions and len(request.instructions) > self._max_instructions_length: - return f"Instructions too long (max {self._max_instructions_length} characters)" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]: """Resolve ref_audio to (wav_samples, sample_rate). @@ -2573,9 +2098,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # ---- Ming TTS helpers ---- - def _is_ming_tts_model(self) -> bool: - return self._tts_model_type == "ming_tts" - def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate): from torchaudio.functional import resample as resample_audio @@ -3488,6 +3010,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self, request: OpenAICreateSpeechRequest, request_id: str | None = None, + has_inline_ref_audio: bool | None = None, ) -> tuple[str, Any, dict[str, Any]]: if self.engine_client.errored: raise self.engine_client.dead_error @@ -3520,7 +3043,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # in place. The builders need to know whether the caller supplied audio # inline vs. via an uploaded voice. model_type: str | None = None - has_inline_ref_audio = request.ref_audio is not None + has_inline_ref_audio = (request.ref_audio is not None) if has_inline_ref_audio is None else has_inline_ref_audio if self._tts_model_type == "ming_flash_omni_tts": # ming_flash_omni is intentionally NOT migrated onto the adapter # framework in this PR (it has no registered adapter); keep it on the @@ -3743,12 +3266,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): base64_encode: bool = False, request_id: str | None = None, usage_out: list[SpeechTokenUsage] | None = None, + has_inline_ref_audio: bool | None = None, ) -> tuple[bytes | str, str]: # ``usage_out`` is an opt-in output channel: when a list is passed, the # computed SpeechTokenUsage is appended to it. The return stays a # 2-tuple so existing callers (and their test mocks) are unaffected; # only the batch path, which surfaces per-item usage, opts in. - request_id, generator, bytes_tts_params = await self._prepare_speech_generation(request, request_id=request_id) + request_id, generator, bytes_tts_params = await self._prepare_speech_generation( + request, request_id=request_id, has_inline_ref_audio=has_inline_ref_audio + ) artifact_ready = False try: @@ -4205,17 +3731,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): merged_requests = [self._merge_batch_item(batch_request, item) for item in batch_request.items] async def _run_item(idx: int, req: OpenAICreateSpeechRequest) -> SpeechBatchItemResult: - # Batch validation still goes through _validate_tts_request directly - # (not the adapter). The single-request path validates via the - # adapter; both ultimately call the same per-model validators, so the - # surfaces stay in sync. Routing batch through the adapter (and its - # uploaded-speaker handling) is a follow-up (RFC #4327). + has_inline_ref_audio = req.ref_audio is not None validation_error = self._validate_tts_request(req) if validation_error is not None: return SpeechBatchItemResult(index=idx, status="error", error=validation_error) usage_box: list[SpeechTokenUsage] = [] try: - audio_data, media_type = await self._generate_audio_bytes(req, base64_encode=True, usage_out=usage_box) + audio_data, media_type = await self._generate_audio_bytes( + req, base64_encode=True, usage_out=usage_box, has_inline_ref_audio=has_inline_ref_audio + ) except Exception as e: logger.exception("Batch item %d failed: %s", idx, e) return SpeechBatchItemResult(index=idx, status="error", error=str(e)) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index ad753a91e..58c135044 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -104,6 +104,10 @@ class TTSModelAdapter(ABC): #: Serving backend: ``"ar"`` (engine_client) or ``"diffusion"``. backend: ClassVar[str] = "ar" + max_new_tokens_min = 1 + + max_new_tokens_max = 4096 + def __init__(self, ctx: SpeechServingContext) -> None: self.ctx = ctx diff --git a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py index 6a7a89702..85648442f 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py @@ -16,10 +16,32 @@ class CosyVoice3Adapter(ARTTSAdapter): name = "cosyvoice3" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate CosyVoice3 request parameters. Returns error message or None.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_cosyvoice3_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # CosyVoice3 requires reference audio for voice cloning + if request.ref_audio is None: + return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)" + + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if not request.ref_text or not request.ref_text.strip(): + return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py index f98f89619..70673ac54 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py @@ -15,6 +15,11 @@ class CovoAudioAdapter(ARTTSAdapter): stage_keys = frozenset({"fused_thinker_talker"}) name = "covo_audio" + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + return None + async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index d216117b7..e8080a427 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -16,10 +16,28 @@ class FishSpeechAdapter(ARTTSAdapter): name = "fish_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate Fish Speech request parameters. Returns error message or None.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_fish_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not request.ref_text or not request.ref_text.strip(): + return "Voice cloning requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index e9e8700e6..ddfd194e2 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -16,10 +16,28 @@ class GlmTTSAdapter(ARTTSAdapter): name = "glm_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate GLM-TTS request — requires ref_audio for voice cloning.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_glm_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.ref_audio is None: + return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning" + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not request.ref_text or not request.ref_text.strip(): + return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be >= {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py index 606b4bca1..99949d8b3 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py @@ -16,10 +16,76 @@ class HiggsAudioV2Adapter(ARTTSAdapter): name = "higgs_audio_v2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate higgs_audio_v2 request parameters. Returns error message or None. + + Accepted: plain text -> speech, or shallow voice clone via ``ref_audio`` + + ``ref_text`` (both required together). Still out of scope: preset + ``voice``/``speaker`` selection, ``x_vector_only_mode`` / + ``speaker_embedding`` helpers, ``task_type``/``language``/ + ``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]`` + tags inside the input body. + """ + from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( + MULTI_SPEAKER_TAG_PATTERN, + ) + + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_higgs_audio_v2_request(request) + + if not request.input or not request.input.strip(): + return "higgs_audio_v2: input text cannot be empty" + + # Voice clone: ref_audio and ref_text must come together. + if request.ref_audio is not None and not request.ref_text: + return ( + "higgs_audio_v2 voice clone requires both 'ref_audio' and " + "'ref_text'; received ref_audio without ref_text" + ) + if request.ref_text and request.ref_audio is None: + return ( + "higgs_audio_v2 voice clone requires both 'ref_audio' and " + "'ref_text'; received ref_text without ref_audio" + ) + + if request.x_vector_only_mode is not None: + return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)" + if request.speaker_embedding is not None: + return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)" + if request.voice and request.ref_audio is None: + # _apply_uploaded_speaker runs before this validator; if voice was + # an uploaded speaker, ref_audio is now populated and ref_text is + # backfilled from the speaker entry. A bare voice= with no + # ref_audio means the name didn't resolve to an uploaded speaker + # (and higgs has no built-in preset voices). + return ( + "higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; " + f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. " + f"Got voice={request.voice!r}" + ) + if request.instructions: + return ( + "higgs_audio_v2 v1 does not support 'instructions' (voice " + "style/emotion control); supply plain text instead" + ) + if request.task_type is not None: + return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech" + if request.language is not None: + return ( + "higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text" + ) + if request.speed is not None and request.speed != 1.0: + return ( + "higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)" + ) + + if MULTI_SPEAKER_TAG_PATTERN.search(request.input): + return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py index dde5e0815..9cae9aa2d 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py @@ -16,10 +16,20 @@ class HiggsAudioV3Adapter(ARTTSAdapter): name = "higgs_audio_v3" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate higgs_audio_v3 request parameters.""" + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_higgs_audio_v3_request(request) + if not request.input or not request.input.strip(): + return "higgs_audio_v3: input text cannot be empty" + if request.ref_audio is not None and not request.ref_text: + # Voice clone ref_text is optional for v3 (improves fidelity but not required) + pass + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index 7e1a8e66b..c736afb51 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -9,6 +9,10 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +from vllm.logger import init_logger + +logger = init_logger(__name__) + @register_tts_adapter class MingTTSAdapter(ARTTSAdapter): @@ -16,7 +20,13 @@ class MingTTSAdapter(ARTTSAdapter): name = "ming_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_ming_tts_request(request) + """Validate Ming TTS request parameters. Returns error message or None.""" + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if isinstance(request.ref_audio, list): + return self._validate_ming_tts_podcast_request(request) + return self._validate_ming_tts_single_speaker_request(request) async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool @@ -51,3 +61,82 @@ class MingTTSAdapter(ARTTSAdapter): tts_params = prompt.get("additional_information", {}) # Ming stop-token / max_tokens sampling stays in the orchestrator tail. return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts") + + def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None: + server = self.ctx.server + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if request.speaker_embedding is not None: + if not request.speaker_embedding: + return "'speaker_embedding' must be a non-empty list of floats" + emb_len = len(request.speaker_embedding) + if emb_len != 192: + logger.warning( + "speaker_embedding has %d dimensions; Ming dense expects 192. " + "Wrong dimensions will likely fail or degrade output.", + emb_len, + ) + + voice_lower = request.voice.lower() if isinstance(request.voice, str) else None + uploaded_voice = bool(voice_lower and voice_lower in server.uploaded_speakers) + clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice + + if request.task_type == "Base" and not clone_source_present: + return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample" + + if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip(): + return "'ref_text' must be non-empty when provided with 'ref_audio'" + + if request.ref_text is not None and request.ref_audio is None and not uploaded_voice: + return "'ref_text' requires 'ref_audio' or an uploaded voice sample" + + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None + + def _validate_ming_tts_podcast_request(self, request: "OpenAICreateSpeechRequest") -> str | None: + server = self.ctx.server + if len(request.ref_audio) < 2: + return "Podcast-style Ming requests require at least two 'ref_audio' clips" + + for ref_audio in request.ref_audio: + fmt_err = server._validate_ref_audio_format(ref_audio) + if fmt_err: + return fmt_err + + if not request.ref_text or not request.ref_text.strip(): + return "Podcast-style Ming requests require non-empty 'ref_text'" + + if request.speaker_embedding is not None: + embeddings = request.speaker_embedding + embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1 + if embedding_count != len(request.ref_audio): + return ( + "Podcast-style Ming requests require one speaker embedding per ref_audio clip; " + f"got {embedding_count} embeddings for {len(request.ref_audio)} clips" + ) + if embeddings and isinstance(embeddings[0], list): + for item in embeddings: + if len(item) != 192: + return "Podcast-style Ming speaker embeddings must each have 192 dimensions" + + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 36d98c2d9..ad43a1c01 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -18,10 +18,69 @@ if TYPE_CHECKING: class _MossTTSAdapterBase(ARTTSAdapter): def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - err = self.ctx.server._apply_uploaded_speaker(request) + """Validate any MOSS-TTS-family request (nano + 5 full variants). + + Dispatches by ``self._moss_variant``: + - ``tts``/``realtime``: require ``ref_audio`` (voice cloning). + - ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2`` + optional (defaults to the same ref for both speakers). + - ``sound_effect``: require ``ambient_sound`` (no ref_audio). + - ``voice_generator``: require ``instructions`` (no ref_audio). + - For the legacy moss_tts_nano model_type the variant is None and + we fall through to the original nano contract (ref_audio only). + """ + server = self.ctx.server + err = server._apply_uploaded_speaker(request) if err: return err - return self.ctx.server._validate_moss_tts_request(request) + + if not request.input or not request.input.strip(): + # SoundEffect can legitimately have empty input (just ambient_sound). + if server._moss_variant != "sound_effect": + return "Input text cannot be empty" + + v = server._moss_variant + if v in (None, "tts", "realtime", "local"): + if request.ref_audio is None: + label = ( + "MOSS-TTS-Nano" + if v is None + else ( + "MOSS-TTS-Realtime" + if v == "realtime" + else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS") + ) + ) + return f"{label} requires 'ref_audio' (reference audio for voice cloning)." + return server._validate_ref_audio_format(request.ref_audio) + + if v == "ttsd": + if request.ref_audio is None: + return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)." + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if request.ref_audio_2 is not None: + return server._validate_ref_audio_format(request.ref_audio_2) + return None + + if v == "sound_effect": + if not request.ambient_sound or not request.ambient_sound.strip(): + return ( + "MOSS-SoundEffect requires 'ambient_sound' (natural language " + "description of the sound effect to synthesise)." + ) + return None + + if v == "voice_generator": + if not request.instructions or not request.instructions.strip(): + return ( + "MOSS-VoiceGenerator requires 'instructions' (natural language " + "voice description, e.g. 'a warm female voice with an American accent')." + ) + return None + + return None # unreachable async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index 0f62a169c..c0be8c685 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -1,18 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -"""Qwen3-TTS serving adapter. - -First model migrated onto the adapter framework (RFC #4327). It is the dispatch -surface for Qwen3-TTS: the serving orchestrator routes prompt/param building -through ``build()`` instead of the inline ``else`` branch in -``_prepare_speech_generation``. - -To stay behaviour-identical and avoid divergence from the batch path (which -shares ``_validate_qwen_tts_request`` and ``_build_tts_params``), the adapter -reuses the single-source helper implementations on the serving instance through -``ctx.server`` rather than copying them. Subsequent PRs relocate those bodies as -more models migrate and the batch path also routes through adapters. -""" +"""Qwen3-TTS serving adapter.""" +from pathlib import Path from typing import TYPE_CHECKING from vllm.logger import init_logger @@ -36,13 +25,133 @@ class Qwen3TTSAdapter(ARTTSAdapter): def normalize(self, request: "OpenAICreateSpeechRequest") -> None: """Qwen3-TTS normalization (Base-task inference, voice lowercasing) is performed inside ``validate`` today; kept fused for a strict behaviour - match. See ``_validate_qwen_tts_request``.""" + match.""" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - # Go through the shared dispatcher (which routes to - # _validate_qwen_tts_request for this model-type) rather than the leaf - # validator directly, matching the pre-adapter qwen3 path. - return self.ctx.server._validate_tts_request(request) + """Validate Qwen TTS request parameters. Returns error message or None.""" + # Infer Base task when ref_audio or ref_text is provided without explicit task_type. + server = self.ctx.server + if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None): + request.task_type = "Base" + + # Normalize voice to lowercase for case-insensitive matching + if request.voice is not None: + request.voice = request.voice.lower() + if request.task_type is None and request.voice in server.precomputed_speakers: + request.task_type = "Base" + task_type = request.task_type or "CustomVoice" + + # Validate input is not empty + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Validate language (case-insensitive; normalized to the title-cased config form) + if request.language is not None: + request.language = request.language.title() + if request.language not in server.supported_languages: + return ( + f"Invalid language '{request.language}'. Supported: {', '.join(sorted(server.supported_languages))}" + ) + + # Validate speaker for CustomVoice task + if task_type == "CustomVoice": + if not server.supported_speakers: + return ( + "This model does not support CustomVoice task (no speakers configured). " + "Use task_type='Base' with ref_audio/ref_text for voice cloning, " + "or use a CustomVoice model." + ) + if request.voice is not None and request.voice not in server.supported_speakers: + return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}" + + # Validate speaker_embedding constraints + if request.speaker_embedding is not None: + if task_type != "Base": + return "'speaker_embedding' is only valid for Base task" + if not request.speaker_embedding: + return "'speaker_embedding' must be a non-empty list of floats" + # speaker_embedding implies x_vector_only_mode — set it before + # Base task validation so callers don't need to pass it explicitly. + request.x_vector_only_mode = True + emb_len = len(request.speaker_embedding) + dim_err = server._validate_qwen_tts_speaker_embedding_dim(emb_len) + if dim_err is not None: + return dim_err + # Validate Base task requirements + if task_type == "Base": + if request.voice is None: + # 1. Ensure a voice source is provided + if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None: + return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning" + # 2. Validate ref_audio format if it exists (using the helper from main) + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + # 3. Validate text requirements based on the mode + if not getattr(request, "x_vector_only_mode", False): + if not request.ref_text or not request.ref_text.strip(): + return ( + "Base task requires non-empty 'ref_text' (transcript of " + "the reference audio) unless 'x_vector_only_mode' is enabled" + ) + else: + voice_lower = request.voice.lower() + if voice_lower in server.uploaded_speakers: + # Check if data file exists for uploaded speaker + speaker_info = server.uploaded_speakers[voice_lower] + file_path = Path(speaker_info["file_path"]) + if not file_path.exists(): + return f"Data file for uploaded speaker '{request.voice}' not found on disk" + elif voice_lower in server.precomputed_speakers: + profile = server.precomputed_speakers[voice_lower] + mode = str(profile.get("mode") or "xvec").lower() + ref_text = request.ref_text or profile.get("ref_text") + if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()): + return ( + f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in " + "the request or custom voice manifest" + ) + else: + # need ref_audio for built-in speaker + if request.ref_audio is None: + return ( + f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning" + ) + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + if not getattr(request, "x_vector_only_mode", False) and ( + not request.ref_text or not request.ref_text.strip() + ): + return ( + "Base task requires non-empty 'ref_text' (transcript of " + "the reference audio) unless 'x_vector_only_mode' is enabled" + ) + + # Validate cross-parameter dependencies + if task_type != "Base": + if request.ref_text is not None: + return "'ref_text' is only valid for Base task" + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is only valid for Base task" + + # Validate VoiceDesign task requirements + if task_type == "VoiceDesign" and not request.instructions: + return "VoiceDesign task requires 'instructions' to describe the voice" + + # Validate instructions length (using cached value from initialization) + if request.instructions and len(request.instructions) > server._max_instructions_length: + return f"Instructions too long (max {server._max_instructions_length} characters)" + + # Validate max_new_tokens range + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py index dd1365204..2c720d8e1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py @@ -23,8 +23,9 @@ class StepAudio2Adapter(ARTTSAdapter): name = "step_audio2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - # Dispatcher routes to the step_audio2 case (non-empty input check). - return self.ctx.server._validate_tts_request(request) + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index 221e2f64b..361aa2522 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -19,7 +19,25 @@ class VoxCPM2Adapter(ARTTSAdapter): name = "voxcpm2" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_voxcpm2_request(request) + """Validate VoxCPM2 request parameters. Returns error message or None.""" + server = self.ctx.server + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.voice is not None: + request.voice = request.voice.lower() + available_voices = set(server.uploaded_speakers) | set(server.precomputed_speakers) | {"default"} + if request.voice not in available_voices: + supported = ", ".join(sorted(available_voices)) or "none" + return f"Invalid voice '{request.voice}'. Supported: {supported}" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index 72e18dc84..0b2d400b1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -16,7 +16,32 @@ class VoxtralTTSAdapter(ARTTSAdapter): name = "voxtral_tts" def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: - return self.ctx.server._validate_voxtral_tts_request(request) + """Validate Voxtral TTS request parameters. Returns error message or None.""" + server = self.ctx.server + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Voxtral TTS requires either a preset voice or ref_audio for voice cloning. + if request.voice is None and request.ref_audio is None: + return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided" + + if request.ref_audio is not None: + fmt_err = server._validate_ref_audio_format(request.ref_audio) + if fmt_err: + return fmt_err + + if request.voice is not None: + request.voice = request.voice.lower() + if server.supported_speakers and request.voice not in server.supported_speakers: + return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}" + + if request.max_new_tokens is not None: + if request.max_new_tokens < self.max_new_tokens_min: + return f"max_new_tokens must be at least {self.max_new_tokens_min}" + if request.max_new_tokens > self.max_new_tokens_max: + return f"max_new_tokens cannot exceed {self.max_new_tokens_max}" + + return None async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool