From 11f633aa8338ba968e0e358d4b3a517f652e33e5 Mon Sep 17 00:00:00 2001 From: boatman <1930807094@qq.com> Date: Mon, 10 Aug 2026 09:47:09 +0800 Subject: [PATCH] [Refactor] Add Ming Flash Omni TTS adapter (#5746) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> --- .../openai_api/test_serving_speech.py | 57 +++++++++++++++++ .../openai_api/test_tts_adapter.py | 6 -- .../entrypoints/openai/serving_speech.py | 51 +-------------- .../openai/tts_adapters/__init__.py | 1 + .../tts_adapters/ming_flash_omni_tts.py | 63 +++++++++++++++++++ 5 files changed, 122 insertions(+), 56 deletions(-) create mode 100644 vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index d36bc84be..e1efd9286 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -3602,6 +3602,63 @@ class TestGLMTTSServing: load_tokenizer.assert_called_once() +@pytest.fixture +def ming_flash_omni_tts_server(mocker: MockerFixture): + mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) + mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock( + model="inclusionAI/Ming-omni-tts-0.5B", + ) + mock_engine_client.default_sampling_params_list = [ + SimpleNamespace(max_tokens=2048, min_tokens=None, extra_args=None) + ] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace(model_stage="ming_tts"), + tts_args={}, + ) + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + + +class TestMingFlashOmniTTSServing: + def test_validate_ming_flash_omni_tts_rejects_ref_text(self, ming_flash_omni_tts_server): + request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript") + error = ming_flash_omni_tts_server._validate_tts_request(request) + assert error is not None + assert "ref_text" in error + + def test_validate_ming_flash_omni_tts_rejects_ref_audio(self, ming_flash_omni_tts_server): + request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") + error = ming_flash_omni_tts_server._validate_tts_request(request) + assert error is not None + assert "ref_audio" in error + + def test_ming_flash_omni_tts_adapter_builds_prompt(self, ming_flash_omni_tts_server, mocker: MockerFixture): + ming_flash_omni_tts_server._build_ming_flash_omni_prompt = mocker.MagicMock( + return_value={ + "prompt_token_ids": [1, 2, 3], + "additional_information": {"voice": ["test"]}, + } + ) + request = OpenAICreateSpeechRequest(input="Hello", voice="test") + asyncio.run(ming_flash_omni_tts_server._prepare_speech_generation(request)) + ming_flash_omni_tts_server._build_ming_flash_omni_prompt.assert_called_once() + + class TestTTSAsyncOffloading: """Tests for event-loop-safe offloading of blocking TTS operations.""" diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 5f6d5930a..89106900b 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -63,12 +63,6 @@ def test_resolve_unknown_returns_none(): assert resolve_adapter(None) is None -def test_ming_flash_omni_not_migrated(): - """ming_flash_omni is intentionally excluded from the adapter migration in - this PR; it stays on the legacy inline dispatch in serving_speech.py.""" - assert resolve_adapter("ming_flash_omni_tts") is None - - def test_voxcpm2_resolves(): """VoxCPM2 (the served ``latent_generator`` model) resolves cleanly. diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index abb7d939e..288625dfe 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1447,9 +1447,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate TTS request parameters. Returns error message or None.""" - if self._tts_model_type == "ming_flash_omni_tts": - return self._validate_ming_flash_omni_tts_request(request) - adapter = self._get_tts_adapter() if adapter is not None: return adapter.validate(request) @@ -1483,41 +1480,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ids = self._voxcpm2_tokenizer.encode(text, add_special_tokens=True) return split_multichar_chinese(ids, self._voxcpm2_split_map) - def _validate_ming_flash_omni_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate Ming-flash-omni standalone-talker request parameters.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - if request.instructions is not None: - if not isinstance(request.instructions, str): - return "instructions must be a string" - if len(request.instructions) > self._max_instructions_length: - return f"instructions exceeds max length {self._max_instructions_length}" - - if request.task_type is not None: - return "'task_type' is not supported for Ming-flash-omni TTS" - if request.language is not None: - return "'language' is not supported for Ming-flash-omni TTS (language is inferred from input text)" - if request.x_vector_only_mode is not None: - return "'x_vector_only_mode' is not supported for Ming-flash-omni TTS" - if request.initial_codec_chunk_frames is not None: - return "'initial_codec_chunk_frames' is not supported for Ming-flash-omni TTS" - - # Per-request voice cloning from raw audio is not yet wired up: Ming - # extracts spk_emb / prompt_wav_lat / prompt_wav_emb model-side via - # register_prompt_wav() at engine init. For ad-hoc cloning, callers - # should pre-compute speaker_embedding and pass it directly. - if request.ref_audio is not None: - return ( - "'ref_audio' is not yet supported for Ming-flash-omni TTS; " - "use a preset 'voice' or 'speaker_embedding' instead" - ) - if request.ref_text is not None: - return "'ref_text' is not yet supported for Ming-flash-omni TTS" - - if request.max_new_tokens is not None and request.max_new_tokens <= 0: - return "'max_new_tokens' must be a positive integer" - return None - def _validate_ref_audio_format(self, ref_audio: str) -> str | None: """Validate ref_audio is a supported URI format. Returns error or None.""" if not isinstance(ref_audio, str): @@ -3173,18 +3135,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # inline vs. via an uploaded voice. model_type: str | None = None has_inline_ref_audio = (request.ref_audio is not None) if has_inline_ref_audio is None else has_inline_ref_audio - if self._tts_model_type == "ming_flash_omni_tts": - # ming_flash_omni is intentionally NOT migrated onto the adapter - # framework in this PR (it has no registered adapter); keep it on the - # legacy inline dispatch so serving still works. - model_type = "ming_flash_omni_tts" - validation_error = self._validate_ming_flash_omni_tts_request(request) - if validation_error: - raise ValueError(validation_error) - prompt = self._build_ming_flash_omni_prompt(request) - tts_params = {} - qwen3_ref_audio_warmup_artifact_key = None - elif (adapter := self._get_tts_adapter()) is not None: + if (adapter := self._get_tts_adapter()) is not None: validation_error = adapter.validate(request) if validation_error: raise ValueError(validation_error) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index f51fa4140..45f7da06c 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -128,6 +128,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 higgs_audio_v2, higgs_audio_v3, indextts2, + ming_flash_omni_tts, ming_tts, moss_tts, omnivoice, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py new file mode 100644 index 000000000..f351e7531 --- /dev/null +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MingFlashOmniTTS serving adapter.""" + +from typing import TYPE_CHECKING + +from vllm.logger import init_logger + +from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest + +if TYPE_CHECKING: + from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest + +logger = init_logger(__name__) + + +@register_tts_adapter +class MingFlashOmniTTSAdapter(ARTTSAdapter): + stage_keys = frozenset({"ming_tts"}) + name = "ming_flash_omni_tts" + + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: + """Validate Ming-flash-omni standalone-talker request parameters.""" + server = self.ctx.server + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + if request.instructions is not None: + if not isinstance(request.instructions, str): + return "instructions must be a string" + if len(request.instructions) > server._max_instructions_length: + return f"instructions exceeds max length {server._max_instructions_length}" + + if request.task_type is not None: + return "'task_type' is not supported for Ming-flash-omni TTS" + if request.language is not None: + return "'language' is not supported for Ming-flash-omni TTS (language is inferred from input text)" + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is not supported for Ming-flash-omni TTS" + if request.initial_codec_chunk_frames is not None: + return "'initial_codec_chunk_frames' is not supported for Ming-flash-omni TTS" + + # Per-request voice cloning from raw audio is not yet wired up: Ming + # extracts spk_emb / prompt_wav_lat / prompt_wav_emb model-side via + # register_prompt_wav() at engine init. For ad-hoc cloning, callers + # should pre-compute speaker_embedding and pass it directly. + if request.ref_audio is not None: + return ( + "'ref_audio' is not yet supported for Ming-flash-omni TTS; " + "use a preset 'voice' or 'speaker_embedding' instead" + ) + if request.ref_text is not None: + return "'ref_text' is not yet supported for Ming-flash-omni TTS" + + if request.max_new_tokens is not None and request.max_new_tokens <= 0: + return "'max_new_tokens' must be a positive integer" + return None + + async def build( + self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool + ) -> PreparedRequest: + server = self.ctx.server + prompt = server._build_ming_flash_omni_prompt(request) + return PreparedRequest(prompt=prompt, tts_params={}, model_type="ming_flash_omni_tts")