diff --git a/tests/entrypoints/openai_api/test_audex_serving_guards.py b/tests/entrypoints/openai_api/test_audex_serving_guards.py index 27ed11356..87524825e 100644 --- a/tests/entrypoints/openai_api/test_audex_serving_guards.py +++ b/tests/entrypoints/openai_api/test_audex_serving_guards.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Serving-layer regressions for the Audex zero-codec-token contract. A zero-codec-token (or phase-invalid TTA) request reaches the serving layer @@ -171,7 +171,7 @@ class TestAudexCfgValidation: class TestAudexCfgInjection: def _serving_with_fake_tokenizer(self): serving = _serving() - serving._get_audex_tokenizer = lambda: _MarkerTokenizer() + serving._adapter._get_tokenizer = lambda: _MarkerTokenizer() return serving def _cond_prompt(self, text: str = "Hello world.") -> str: @@ -333,8 +333,8 @@ class TestAdapterRouting: class TestAudexTTAInjection: def _serving(self): serving = _serving(model_type="audex_tta") - serving._get_audex_tokenizer = lambda: _TTAVocabTokenizer() - serving._audex_tta_rvq = None + serving._adapter._get_tokenizer = lambda: _TTAVocabTokenizer() + serving._adapter._tta_rvq = None return serving def _prompt(self): diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 47c11d740..7e1a9ccec 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -53,7 +53,7 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ( ) from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import MingTTSAdapter -from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import Qwen3TTSCodecLimitError +from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import Qwen3TTSAdapter, Qwen3TTSCodecLimitError from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( FISH_TEXT_ONLY_SYSTEM_PROMPT, @@ -812,6 +812,15 @@ class TestTTSMethods: models=mock_models, request_logger=mocker.MagicMock(), ) + qwen3_adapter = Qwen3TTSAdapter( + SpeechServingContext( + server=server, + engine_client=mock_engine_client, + ) + ) + server._build_tts_params = qwen3_adapter._build_tts_params + server._estimate_prompt_len = qwen3_adapter._estimate_prompt_len + yield server server.shutdown() @@ -1663,12 +1672,12 @@ class TestTTSMethods: def test_conditioning_cache_salt_changes_with_ref_audio_cache_key(self): """Prefix-cache salt must fold the content-aware resolve key so a same-path file rewrite cannot reuse KV from the previous reference.""" - from vllm_omni.entrypoints.openai.serving_speech import _conditioning_cache_salt + from vllm_omni.entrypoints.openai.tts_adapters.base import conditioning_cache_salt req = OpenAICreateSpeechRequest(input="hello", ref_audio="file:///data/spk.wav") - salt_a = _conditioning_cache_salt(req, {"ref_audio_cache_key": "key_aaa"}) - salt_b = _conditioning_cache_salt(req, {"ref_audio_cache_key": "key_bbb"}) - salt_locator_only = _conditioning_cache_salt(req, {}) + salt_a = conditioning_cache_salt(req, {"ref_audio_cache_key": "key_aaa"}) + salt_b = conditioning_cache_salt(req, {"ref_audio_cache_key": "key_bbb"}) + salt_locator_only = conditioning_cache_salt(req, {}) assert salt_a != salt_b assert salt_a != salt_locator_only @@ -1689,20 +1698,22 @@ class TestTTSMethods: } adapter = HiggsAudioV3TokenizerAdapter(tokenizer) mocker.patch.object(adapter, "build_prompt", return_value=[1, -100, -100, 2]) - speech_server._resolve_higgs_audio_v3_adapter = mocker.AsyncMock(return_value=adapter) + speech_server._tts_model_type = "higgs_audio_v3" + speech_server._adapter = speech_server._get_tts_adapter() + speech_server._adapter._resolve_higgs_audio_v3_adapter = mocker.AsyncMock(return_value=adapter) codes = torch.arange(16, dtype=torch.long).view(2, 8) - speech_server._resolve_higgs_audio_v3_ref_codes = mocker.AsyncMock(return_value=(codes, False, False)) - speech_server._get_resolved_ref_audio_artifact_key = mocker.MagicMock(return_value="art") + speech_server._adapter._resolve_higgs_audio_v3_ref_codes = mocker.AsyncMock(return_value=(codes, False, False)) + speech_server._adapter._get_resolved_ref_audio_artifact_key = mocker.MagicMock(return_value="art") req = OpenAICreateSpeechRequest( input="hello", ref_audio="file:///data/spk.wav", ref_text="transcript", ) - speech_server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.1] * 48000, 24000, "key_aaa")) - prompt_a = await speech_server._build_higgs_audio_v3_params(req) - speech_server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.9] * 48000, 24000, "key_bbb")) - prompt_b = await speech_server._build_higgs_audio_v3_params(req) + speech_server._adapter._resolve_ref_audio = mocker.AsyncMock(return_value=([0.1] * 48000, 24000, "key_aaa")) + prompt_a = await speech_server._adapter._build_higgs_audio_v3_params(req) + speech_server._adapter._resolve_ref_audio = mocker.AsyncMock(return_value=([0.9] * 48000, 24000, "key_bbb")) + prompt_b = await speech_server._adapter._build_higgs_audio_v3_params(req) assert prompt_a["prompt_token_ids"] == [1, 151700, 151700, 2] assert prompt_a["prompt_token_ids"] == prompt_b["prompt_token_ids"] @@ -1830,7 +1841,9 @@ class TestTTSMethods: @pytest.mark.asyncio async def test_ttsd_second_reference_does_not_use_named_voice(self, speech_server, mocker): """Uploaded voice names speaker 1 only; speaker 2 must stay anonymous.""" - speech_server._moss_variant = "ttsd" + speech_server._tts_model_type = "moss_tts" + speech_server._adapter = speech_server._get_tts_adapter() + speech_server._adapter._moss_variant = "ttsd" speech_server.uploaded_speakers = {"alice": {}} mocker.patch.object(speech_server, "_voice_created_at", return_value=42) @@ -1839,7 +1852,7 @@ class TestTTSMethods: proc.model_config.sampling_rate = 24000 proc.build_user_message.return_value = "msg" proc.return_value = {"input_ids": [torch.zeros((4, 9), dtype=torch.int64)]} - mocker.patch.object(speech_server, "_get_moss_processor", return_value=proc) + mocker.patch.object(speech_server._adapter, "_get_moss_processor", return_value=proc) seen: list[tuple[str, str | None]] = [] @@ -1848,7 +1861,7 @@ class TestTTSMethods: seen.append((ref_str, kwargs.get("voice_name"))) return torch.zeros(3, dtype=torch.int64), f"rk:{ref_str}" - mocker.patch.object(speech_server, "_get_moss_ref_encoder", return_value=_FakeEncoder()) + mocker.patch.object(speech_server._adapter, "_get_moss_ref_encoder", return_value=_FakeEncoder()) req = OpenAICreateSpeechRequest( input="hello", @@ -1856,7 +1869,7 @@ class TestTTSMethods: ref_audio="data:audio/wav;base64,aaa", ref_audio_2="data:audio/wav;base64,bbb", ) - params = await speech_server._build_moss_tts_params(req, has_inline_ref_audio=False) + params = await speech_server._adapter._build_moss_tts_params(req, has_inline_ref_audio=False) assert sorted(seen) == [ ("data:audio/wav;base64,aaa", "alice"), ("data:audio/wav;base64,bbb", None), @@ -1887,7 +1900,7 @@ class TestTTSMethods: assert speech_server._validate_tts_request(req) is None assert req.task_type == "Base" - params = speech_server._build_tts_params(req) + params = speech_server._adapter._build_tts_params(req) assert params["task_type"] == ["Base"] assert params["speaker"] == ["alice"] assert params["x_vector_only_mode"] == [False] @@ -2036,14 +2049,14 @@ class TestTTSMethods: ) speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] speech_server.engine_client.generate = mocker.MagicMock(return_value="generator") - speech_server._build_voxcpm2_prompt = mocker.AsyncMock( + speech_server._adapter._build_prompt = mocker.AsyncMock( return_value={"prompt_token_ids": [1], "additional_information": {}} ) with pytest.raises(ValueError, match="Invalid voice 'bob'"): asyncio.run(speech_server._prepare_speech_generation(OpenAICreateSpeechRequest(input="Hello", voice="Bob"))) - speech_server._build_voxcpm2_prompt.assert_not_awaited() + speech_server._adapter._build_prompt.assert_not_awaited() speech_server.engine_client.generate.assert_not_called() def test_prepare_voxcpm2_accepts_default_voice(self, speech_server, mocker): @@ -2057,13 +2070,13 @@ class TestTTSMethods: ) speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(())) - speech_server._build_voxcpm2_prompt = mocker.AsyncMock( + speech_server._adapter._build_prompt = mocker.AsyncMock( return_value={"prompt_token_ids": [1], "additional_information": {}} ) asyncio.run(speech_server._prepare_speech_generation(OpenAICreateSpeechRequest(input="Hello", voice="default"))) - speech_server._build_voxcpm2_prompt.assert_awaited_once() + speech_server._adapter._build_prompt.assert_awaited_once() speech_server.engine_client.generate.assert_called_once() def test_prepare_voxcpm2_precomputed_voice_sets_model_cache_key(self, speech_server, mocker): @@ -2084,7 +2097,7 @@ class TestTTSMethods: ) speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(())) - speech_server._build_voxcpm2_prompt = mocker.AsyncMock( + speech_server._adapter._build_prompt = mocker.AsyncMock( return_value={ "prompt_token_ids": [1], "additional_information": { @@ -2478,13 +2491,14 @@ class TestTTSMethods: speech_server, "_get_uploaded_speaker_embedding", return_value=fake_embedding ) mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data") + adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) + speech_server._adapter = adapter mock_prompt = mocker.patch.object( - speech_server, + speech_server._adapter, "_build_ming_dense_prompt", return_value={"additional_information": {"speaker_count": 1}}, ) - adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) req = OpenAICreateSpeechRequest(input="Hello", voice="emb_voice") prepared = asyncio.run(adapter.build(req, [], False)) @@ -2522,13 +2536,14 @@ class TestTTSMethods: "_resolve_ref_audio", new=mocker.AsyncMock(return_value=(*ref_audio_data, "fake_cache_key")), ) + adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) + speech_server._adapter = adapter mock_prompt = mocker.patch.object( - speech_server, + speech_server._adapter, "_build_ming_dense_prompt", return_value={"additional_information": {"speaker_count": 1}}, ) - adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) req = OpenAICreateSpeechRequest(input="Hello", voice="audio_voice") prepared = asyncio.run(adapter.build(req, [], False)) @@ -2552,6 +2567,8 @@ class TestTTSMethods: def convert_tokens_to_ids(self, token): return 888 if token == "" else self.unk_token_id + speech_server._tts_model_type = "ming_tts" + speech_server._adapter = speech_server._get_tts_adapter() speech_server._tts_tokenizer = FakeTokenizer() request = OpenAICreateSpeechRequest( input="Hello", @@ -2562,7 +2579,7 @@ class TestTTSMethods: ([0.4, 0.5], 24000), ] - prompt = speech_server._build_ming_dense_prompt( + prompt = speech_server._adapter._build_ming_dense_prompt( request, ref_audio_data=ref_audio_data, voice_name="uploaded_voice", @@ -2592,10 +2609,12 @@ class TestTTSMethods: def convert_tokens_to_ids(self, token): return 888 if token == "" else self.unk_token_id + speech_server._tts_model_type = "ming_tts" + speech_server._adapter = speech_server._get_tts_adapter() speech_server._tts_tokenizer = FakeTokenizer() request = OpenAICreateSpeechRequest(input="Hello") - prompt = speech_server._build_ming_dense_prompt( + prompt = speech_server._adapter._build_ming_dense_prompt( request, ref_audio_data=([0.1, 0.2, 0.3], 16000), ) @@ -2617,6 +2636,10 @@ class TestTTSMethods: from safetensors.torch import save_file speech_server._tts_model_type = "ming_tts" + speech_server._adapter = speech_server._get_tts_adapter() + speech_server._adapter.ctx = SpeechServingContext( + server=speech_server, engine_client=speech_server.engine_client + ) speech_server.uploaded_speakers_dir = tmp_path speech_server._speaker_cache.clear() file_path = tmp_path / "legacy_audio_voice.safetensors" @@ -2640,16 +2663,15 @@ class TestTTSMethods: new=mocker.AsyncMock(return_value=(*ref_audio_data, "fake_cache_key")), ) mock_prompt = mocker.patch.object( - speech_server, + speech_server._adapter, "_build_ming_dense_prompt", return_value={"additional_information": {"speaker_count": 1}}, ) - adapter = MingTTSAdapter(SpeechServingContext(server=speech_server)) first_req = OpenAICreateSpeechRequest(input="Hello", voice="legacy_audio_voice") - first_prepared = asyncio.run(adapter.build(first_req, [], False)) + first_prepared = asyncio.run(speech_server._adapter.build(first_req, [], False)) second_req = OpenAICreateSpeechRequest(input="Hello again", voice="legacy_audio_voice") - second_prepared = asyncio.run(adapter.build(second_req, [], False)) + second_prepared = asyncio.run(speech_server._adapter.build(second_req, [], False)) assert first_req.speaker_embedding is None assert second_req.speaker_embedding is None @@ -2921,7 +2943,7 @@ class TestTTSMethods: ) speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(())) - speech_server._build_voxcpm2_prompt = mocker.AsyncMock( + speech_server._adapter._build_prompt = mocker.AsyncMock( return_value={"prompt_token_ids": [1], "additional_information": {}} ) @@ -4182,13 +4204,13 @@ def fish_speech_server(mocker: MockerFixture): class TestFishSpeechServing: def test_build_fish_prompt_normalizes_legacy_speaker_tags(self, fish_speech_server): tokenizer = _FakeFishTokenizer() - fish_speech_server._fish_speech_tokenizer = tokenizer + fish_speech_server._adapter._tokenizer = tokenizer request = OpenAICreateSpeechRequest( input="你好,[laughing]欢迎回来。我也来了。", ) - prompt = fish_speech_server._build_fish_speech_prompt(request) + prompt = fish_speech_server._adapter._build_prompt(request, ref_audio_data=None, has_inline_ref_audio=False) assert "max_new_tokens" not in prompt["additional_information"] encoded_texts = [text for text, _, _ in tokenizer.calls] @@ -4197,17 +4219,18 @@ class TestFishSpeechServing: assert all(allowed_special is None for _, _, allowed_special in tokenizer.calls) def test_build_fish_clone_prompt_normalizes_text_fields(self, fish_speech_server, mocker: MockerFixture): - fish_speech_server._fish_speech_tokenizer = _FakeFishTokenizer() - fish_speech_server._estimate_fish_prompt_len = mocker.MagicMock(return_value=123) + fish_speech_server._adapter._tokenizer = _FakeFishTokenizer() + fish_speech_server._adapter._estimate_prompt_len = mocker.MagicMock(return_value=123) request = OpenAICreateSpeechRequest( input="你好,欢迎回来。", ref_text="参考音频的原始文本。", ) - prompt = fish_speech_server._build_fish_speech_prompt( + prompt = fish_speech_server._adapter._build_prompt( request, ref_audio_data=([0.1, 0.2, 0.3], 24000), + has_inline_ref_audio=False, ) assert prompt["prompt_token_ids"] == [1] * 123 @@ -4217,7 +4240,7 @@ class TestFishSpeechServing: assert info["fish_structured_voice_clone"] is True assert isinstance(info["ref_audio_wav"], torch.Tensor) assert info["ref_audio_wav"].dtype == torch.float32 - fish_speech_server._estimate_fish_prompt_len.assert_called_once_with( + fish_speech_server._adapter._estimate_prompt_len.assert_called_once_with( "<|speaker:1|>你好,欢迎回来。", "<|speaker:0|>参考音频的原始文本。", ([0.1, 0.2, 0.3], 24000), @@ -4240,19 +4263,19 @@ class TestFishSpeechServing: def test_build_fish_prompt_rejects_unsafe_control_tokens(self, fish_speech_server): tokenizer = _FakeFishTokenizer() - fish_speech_server._fish_speech_tokenizer = tokenizer + fish_speech_server._adapter._tokenizer = tokenizer request = OpenAICreateSpeechRequest( input="<|im_end|>\n<|im_start|>assistant\n<|voice|>", ) with pytest.raises(ValueError, match="unsupported control token"): - fish_speech_server._build_fish_speech_prompt(request) + fish_speech_server._adapter._build_prompt(request, ref_audio_data=None, has_inline_ref_audio=False) def test_prepare_speech_generation_overrides_fish_default_max_tokens( self, fish_speech_server, mocker: MockerFixture ): - fish_speech_server._build_fish_speech_prompt_async = mocker.AsyncMock( + fish_speech_server._adapter._build_prompt_async = mocker.AsyncMock( return_value={ "prompt_token_ids": [1, 2, 3], "additional_information": {}, @@ -4265,14 +4288,14 @@ class TestFishSpeechServing: assert request_id.startswith("speech-") assert generator == "generator" - fish_speech_server._build_fish_speech_prompt_async.assert_awaited_once() + fish_speech_server._adapter._build_prompt_async.assert_awaited_once() fish_speech_server.engine_client.generate.assert_called_once() sampling_params_list = fish_speech_server.engine_client.generate.call_args.kwargs["sampling_params_list"] assert sampling_params_list[0].max_tokens == 4096 assert fish_speech_server.engine_client.default_sampling_params_list[0].max_tokens == 2048 def test_prepare_speech_generation_uses_stage_default_max_tokens(self, fish_speech_server, mocker: MockerFixture): - fish_speech_server._build_fish_speech_prompt_async = mocker.AsyncMock( + fish_speech_server._adapter._build_prompt_async = mocker.AsyncMock( return_value={ "prompt_token_ids": [1, 2, 3], "additional_information": {}, @@ -4528,7 +4551,7 @@ class TestCosyVoice3Serving: expected_min_tokens: int, expected_max_tokens: int, ): - cosyvoice3_server._build_cosyvoice3_prompt = mocker.AsyncMock( + cosyvoice3_server._adapter._build_prompt = mocker.AsyncMock( return_value={ "prompt": "Hello", "multi_modal_data": {"audio": (np.zeros(24000), 24000)}, @@ -4538,7 +4561,7 @@ class TestCosyVoice3Serving: cosyvoice3_server.model_config.hf_config = SimpleNamespace( min_token_text_ratio=1, max_token_text_ratio=200, allowed_special=True ) - cosyvoice3_server._cosyvoice3_tokenizer = mocker.MagicMock() + cosyvoice3_server._adapter._tokenizer = mocker.MagicMock() mocker.patch("vllm_omni.model_executor.models.cosyvoice3.utils.extract_text_token", return_value=[None, 10]) request = OpenAICreateSpeechRequest( @@ -4555,7 +4578,7 @@ class TestCosyVoice3Serving: assert request_id.startswith("speech-") assert generator == "generator" assert tts_params == {} - cosyvoice3_server._build_cosyvoice3_prompt.assert_awaited_once() + cosyvoice3_server._adapter._build_prompt.assert_awaited_once() # ---- GLM-TTS Serving Tests ---- @@ -4635,13 +4658,13 @@ class TestGLMTTSServing: return_value="normalized target", ) - text_token_len = glm_tts_server._estimate_glm_tts_text_token_len("abcdef") + text_token_len = glm_tts_server._adapter._estimate_text_token_len("abcdef") assert text_token_len == 3 load_tokenizer.assert_called_once() def test_prepare_speech_generation_glm_tts(self, glm_tts_server, mocker: MockerFixture): - glm_tts_server._build_glm_tts_prompt = mocker.AsyncMock( + glm_tts_server._adapter._build_prompt = mocker.AsyncMock( return_value={ "prompt": "Hello", "multi_modal_data": {"audio": (np.zeros(24000), 24000)}, @@ -4654,7 +4677,7 @@ class TestGLMTTSServing: ref_text="Reference text", ) glm_tts_server.model_config.hf_config = SimpleNamespace(min_token_text_ratio=1, max_token_text_ratio=200) - glm_tts_server._estimate_glm_tts_text_token_len = mocker.MagicMock(return_value=10) + glm_tts_server._adapter._estimate_text_token_len = mocker.MagicMock(return_value=10) request_id, generator, tts_params = asyncio.run(glm_tts_server._prepare_speech_generation(request)) sampling_params = glm_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] @@ -4663,7 +4686,7 @@ class TestGLMTTSServing: assert request_id.startswith("speech-") assert generator == "generator" assert tts_params == {} - glm_tts_server._build_glm_tts_prompt.assert_awaited_once() + glm_tts_server._adapter._build_prompt.assert_awaited_once() @pytest.fixture @@ -4782,16 +4805,14 @@ class TestMingFlashOmniTTSServing: assert error is not None assert "ref_audio" in error - def test_ming_flash_omni_tts_adapter_builds_prompt(self, ming_flash_omni_tts_server, mocker: MockerFixture): - ming_flash_omni_tts_server._build_ming_flash_omni_prompt = mocker.MagicMock( - return_value={ - "prompt_token_ids": [1, 2, 3], - "additional_information": {"voice": ["test"]}, - } - ) + def test_ming_flash_omni_tts_adapter_builds_prompt(self, ming_flash_omni_tts_server): request = OpenAICreateSpeechRequest(input="Hello", voice="test") asyncio.run(ming_flash_omni_tts_server._prepare_speech_generation(request)) - ming_flash_omni_tts_server._build_ming_flash_omni_prompt.assert_called_once() + + prompt = ming_flash_omni_tts_server.engine_client.generate.call_args.kwargs["prompt"] + assert prompt["prompt_token_ids"] == [0] + assert prompt["additional_information"]["text"] == "Hello" + assert prompt["additional_information"]["voice_name"] == "test" @pytest.fixture @@ -4880,10 +4901,6 @@ class TestDotsTTSServing: class TestTTSAsyncOffloading: """Tests for event-loop-safe offloading of blocking TTS operations.""" - def test_build_voxtral_prompt_is_sync(self): - """_build_voxtral_prompt should be a regular function, not a coroutine.""" - assert not asyncio.iscoroutinefunction(OmniOpenAIServingSpeech._build_voxtral_prompt) - @pytest.fixture def voxtral_server(self, mocker: MockerFixture): mocker.patch( @@ -4977,7 +4994,7 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_awaits_voxtral_async(self, voxtral_server, mocker: MockerFixture): """Voxtral path in _prepare_speech_generation should call the async wrapper.""" - voxtral_server._build_voxtral_prompt_async = mocker.AsyncMock( + voxtral_server._adapter._build_prompt_async = mocker.AsyncMock( return_value={ "prompt_token_ids": [1, 2, 3], "additional_information": {"voice": ["test"]}, @@ -4985,19 +5002,19 @@ class TestTTSAsyncOffloading: ) request = OpenAICreateSpeechRequest(input="hello", voice="test") asyncio.run(voxtral_server._prepare_speech_generation(request)) - voxtral_server._build_voxtral_prompt_async.assert_awaited_once() + voxtral_server._adapter._build_prompt_async.assert_awaited_once() def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture): """Qwen3 TTS path should call _estimate_prompt_len_async.""" qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) request = OpenAICreateSpeechRequest(input="hello") asyncio.run(qwen3_tts_server._prepare_speech_generation(request)) - qwen3_tts_server._build_tts_params.assert_called_once() - qwen3_tts_server._estimate_prompt_len_async.assert_awaited_once() + qwen3_tts_server._adapter._build_tts_params.assert_called_once() + qwen3_tts_server._adapter._estimate_prompt_len_async.assert_awaited_once() def test_prepare_speech_generation_qwen3_default_seed_sets_tts_local_seed( self, qwen3_tts_server, mocker: MockerFixture @@ -5007,10 +5024,10 @@ class TestTTSAsyncOffloading: SimpleNamespace(max_tokens=2048, seed=42, extra_args=None) ] qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) request = OpenAICreateSpeechRequest(input="hello") asyncio.run(qwen3_tts_server._prepare_speech_generation(request)) @@ -5058,10 +5075,10 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=sse should request delta-style multimodal outputs.""" qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) mock_coerce = mocker.patch( "vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list, @@ -5075,10 +5092,10 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream=True should request delta-style multimodal outputs for SSE streaming.""" qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) mock_coerce = mocker.patch( "vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list, @@ -5092,10 +5109,10 @@ class TestTTSAsyncOffloading: def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture): """stream_format=audio should request delta-style multimodal outputs.""" qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) mock_coerce = mocker.patch( "vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list, @@ -5112,10 +5129,10 @@ class TestTTSAsyncOffloading: """Full-payload TTS streaming should not request delta multimodal outputs.""" qwen3_tts_server.engine_client.model_config.async_chunk = False qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None) - qwen3_tts_server._build_tts_params = mocker.MagicMock( + qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock( return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]} ) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) mock_coerce = mocker.patch( "vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list, @@ -5132,7 +5149,7 @@ class TestTTSAsyncOffloading: """FINAL_ONLY streaming for async_chunk=False is scoped to qwen3_tts only.""" voxtral_server.engine_client.model_config.async_chunk = False mocker.patch.object(voxtral_server._get_tts_adapter(), "validate", return_value=None) - voxtral_server._build_voxtral_prompt_async = mocker.AsyncMock( + voxtral_server._adapter._build_prompt_async = mocker.AsyncMock( return_value={ "prompt_token_ids": [1, 2, 3], "additional_information": {"voice": ["test"]}, @@ -5153,7 +5170,7 @@ class TestTTSAsyncOffloading: ): """VoiceDesign explicit false should reach the model prompt additional_information.""" qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) request = OpenAICreateSpeechRequest( input="hello", @@ -5176,7 +5193,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) qwen3_tts_server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.0] * 48000, 24000, "fake_cache_key")) qwen3_tts_server._get_resolved_ref_audio_artifact_key = mocker.MagicMock(return_value=None) - qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) + qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) request = OpenAICreateSpeechRequest( input="hello", @@ -5491,7 +5508,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec") icl_params = {"task_type": ["Base"], "x_vector_only_mode": [False]} - assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is False + assert qwen3_tts_server._adapter._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is False def test_qwen3_xvector_ready_artifact_still_reusable_by_xvector_request(self, qwen3_tts_server): qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") @@ -5499,7 +5516,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec") xvec_params = {"task_type": ["Base"], "x_vector_only_mode": [True]} - assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(xvec_params, "artifact-a") is True + assert qwen3_tts_server._adapter._qwen3_tts_can_use_ref_audio_artifact_only(xvec_params, "artifact-a") is True def test_qwen3_icl_ready_artifact_enables_icl_artifact_only(self, qwen3_tts_server): qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a") @@ -5507,7 +5524,7 @@ class TestTTSAsyncOffloading: qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-icl") icl_params = {"task_type": ["Base"], "x_vector_only_mode": [False]} - assert qwen3_tts_server._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is True + assert qwen3_tts_server._adapter._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is True def test_shutdown_is_idempotent(self, mocker: MockerFixture): """Calling shutdown() twice should not raise.""" diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index d1a0174c3..a2c0ace0b 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -22,12 +22,14 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( detect_tts_model_type, resolve_adapter, ) +from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import ( IndexTTS2Adapter, IndexTTS25Adapter, indextts2_conditioning_cache_salt, ) +from vllm_omni.entrypoints.openai.tts_adapters.ming_flash_omni_tts import MingFlashOmniTTSAdapter from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import ( MossTTSAdapter, MossTTSNanoAdapter, @@ -37,6 +39,8 @@ from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import ( Qwen3TTSAdapter, Qwen3TTSCodecLimitError, ) +from vllm_omni.entrypoints.openai.tts_adapters.step_audio2 import StepAudio2Adapter +from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter from vllm_omni.model_executor.models.indextts2 import prompt_utils from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import ( INDEXTTS25_TOKENIZER_FILE, @@ -125,8 +129,8 @@ def test_moss_tts_applies_request_max_new_tokens(adapter_cls): def _build_moss_tts_request(adapter_cls, mocker, *, request_seed): server = mocker.Mock() - server._build_moss_tts_params = mocker.AsyncMock(return_value={}) adapter = adapter_cls(SpeechServingContext(server=server)) + adapter._build_moss_tts_params = mocker.AsyncMock(return_value={}) request = OpenAICreateSpeechRequest(input="hello", seed=request_seed) return asyncio.run( @@ -167,6 +171,81 @@ def test_qwen3_tts_metadata(): assert issubclass(Qwen3TTSAdapter, ARTTSAdapter) +def test_qwen3_tts_build_constructs_prepared_request(): + server = SimpleNamespace(_tts_executor=None, _tts_tokenizer=None, uploaded_speakers={}) + engine_client = SimpleNamespace(model_config=SimpleNamespace()) + adapter = Qwen3TTSAdapter(SimpleNamespace(server=server, engine_client=engine_client)) + + async def estimate_prompt_len(_tts_params): + return 3 + + adapter._estimate_prompt_len_async = estimate_prompt_len + request = OpenAICreateSpeechRequest(input="hello") + + prepared = asyncio.run(adapter.build(request, [], False)) + + assert prepared.prompt["prompt_token_ids"] == [1, 1, 1] + assert prepared.prompt["additional_information"] is prepared.tts_params + assert prepared.tts_params["text"] == ["hello"] + assert prepared.tts_params["speaker"] == ["Vivian"] + assert prepared.model_type == "CustomVoice" + + +def test_covo_audio_build_constructs_tokenized_prompt(): + class FakeTokenizer: + def apply_chat_template(self, messages, *, tokenize, add_generation_prompt): + assert messages[-1] == {"role": "user", "content": "hello"} + assert tokenize is True + assert add_generation_prompt is True + return [11, 12, 13] + + adapter = CovoAudioAdapter(SimpleNamespace(server=SimpleNamespace(), engine_client=SimpleNamespace())) + adapter._tokenizer = FakeTokenizer() + + prepared = asyncio.run(adapter.build(OpenAICreateSpeechRequest(input="hello"), [], False)) + + assert prepared.prompt == {"prompt_token_ids": [11, 12, 13]} + assert prepared.tts_params == {} + assert prepared.model_type == "covo_audio" + + +def test_ming_flash_omni_build_constructs_prepared_request(): + server = SimpleNamespace(uploaded_speakers={}) + adapter = MingFlashOmniTTSAdapter(SimpleNamespace(server=server, engine_client=SimpleNamespace())) + request = OpenAICreateSpeechRequest(input="hello", voice="test") + + prepared = asyncio.run(adapter.build(request, [], False)) + + assert prepared.prompt["prompt_token_ids"] == [0] + info = prepared.prompt["additional_information"] + assert info["ming_task"] == "instruct" + assert info["text"] == "hello" + assert info["voice_name"] == "test" + assert prepared.model_type == "ming_flash_omni_tts" + + +def test_step_audio2_build_constructs_open_assistant_turn(): + adapter = StepAudio2Adapter(SimpleNamespace(server=SimpleNamespace(), engine_client=SimpleNamespace())) + request = OpenAICreateSpeechRequest(input="hello", instructions="Read warmly.") + + prepared = asyncio.run(adapter.build(request, [], False)) + + assert "<|im_start|>system\nRead warmly.<|im_end|>" in prepared.prompt["prompt"] + assert prepared.prompt["prompt"].endswith("<|im_start|>assistant\n") + assert prepared.tts_params == {} + assert prepared.model_type == "step_audio2" + + +def test_voxtral_prompt_builder_is_sync(): + ctx = SimpleNamespace( + server=SimpleNamespace(_tts_executor=None), + engine_client=SimpleNamespace(), + ) + adapter = VoxtralTTSAdapter(ctx) + + assert not asyncio.iscoroutinefunction(adapter._build_prompt) + + @pytest.mark.parametrize( ("task_type", "text_tokens", "request_cap", "expected_cap"), [ diff --git a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py index 3fba046dc..3d43078ce 100755 --- a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py +++ b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py @@ -11,13 +11,15 @@ import asyncio import json import os import time -from collections import OrderedDict, defaultdict +from collections import defaultdict from types import SimpleNamespace import pytest import torch from tests.helpers.stage_config import get_deploy_config_path +from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext +from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v3 import HiggsAudioV3Adapter pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -1683,31 +1685,21 @@ class TestVoiceCloneReferenceCache: assert tok._resolve_audio_tokenizer_dir() is None def test_higgs_v3_ref_code_cache_returns_clone(self): - from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech - - serving = object.__new__(OmniOpenAIServingSpeech) - serving._higgs_audio_v3_ref_code_cache = OrderedDict() - serving._higgs_audio_v3_ref_code_cache_bytes = 0 - serving._higgs_audio_v3_ref_code_inflight = {} + adapter = HiggsAudioV3Adapter(SpeechServingContext(server=SimpleNamespace())) codes = torch.arange(16, dtype=torch.long).reshape(2, 8) - serving._put_higgs_audio_v3_ref_codes("ref-a", codes) + adapter._put_higgs_audio_v3_ref_codes("ref-a", codes) - cached = serving._get_higgs_audio_v3_ref_codes("ref-a") + cached = adapter._get_higgs_audio_v3_ref_codes("ref-a") assert cached is not None cached.fill_(0) - cached_again = serving._get_higgs_audio_v3_ref_codes("ref-a") + cached_again = adapter._get_higgs_audio_v3_ref_codes("ref-a") assert cached_again is not None assert torch.equal(cached_again, codes) def test_higgs_v3_ref_code_inflight_deduplicates_concurrent_encode(self): - from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech - - serving = object.__new__(OmniOpenAIServingSpeech) - serving._higgs_audio_v3_ref_code_cache = OrderedDict() - serving._higgs_audio_v3_ref_code_cache_bytes = 0 - serving._higgs_audio_v3_ref_code_inflight = {} + adapter = HiggsAudioV3Adapter(SpeechServingContext(server=SimpleNamespace())) calls = 0 def encode_reference_audio(_wav, _sr): @@ -1722,7 +1714,7 @@ class TestVoiceCloneReferenceCache: async def run(): return await asyncio.gather( *[ - serving._resolve_higgs_audio_v3_ref_codes( + adapter._resolve_higgs_audio_v3_ref_codes( "ref-a", object(), 24000, @@ -1741,7 +1733,7 @@ class TestVoiceCloneReferenceCache: assert torch.equal(codes, torch.arange(16, dtype=torch.long).reshape(2, 8) + 1) cached, cache_hit, inflight_wait = asyncio.run( - serving._resolve_higgs_audio_v3_ref_codes( + adapter._resolve_higgs_audio_v3_ref_codes( "ref-a", object(), 24000, diff --git a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py index 21c8dfd14..3ec6662cc 100644 --- a/tests/model_executor/models/ming_tts/test_ming_shared_modules.py +++ b/tests/model_executor/models/ming_tts/test_ming_shared_modules.py @@ -1,12 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from types import SimpleNamespace import pytest import torch -from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext +from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import ( + MingTTSAdapter, + parse_ming_instruction_fields, +) from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAEConfig from vllm_omni.model_executor.models.common.ming.cfm_cudagraph import CFMGraphExecutor, CFMSampler @@ -174,19 +178,19 @@ def test_ming_dense_validation_rejects_semantic_audio_vae_config(): def test_ming_instruction_parser_preserves_dense_and_flash_defaults(): """Ming dense and Ming flash keep distinct instruction defaults.""" - serving = object.__new__(OmniOpenAIServingSpeech) - serving.uploaded_speakers = {"uploaded": {}} + serving = SimpleNamespace(uploaded_speakers={"uploaded": {}}) + adapter = MingTTSAdapter(SpeechServingContext(server=serving)) - dense_plain = serving._parse_ming_instruction(SimpleNamespace(instructions="calm", language=None, voice=None)) + dense_plain = adapter._parse_ming_instruction(SimpleNamespace(instructions="calm", language=None, voice=None)) assert dense_plain == "calm" - dense_with_fields = serving._parse_ming_instruction( + dense_with_fields = adapter._parse_ming_instruction( SimpleNamespace(instructions="calm", language="Auto", voice="灵小甄") ) assert dense_with_fields == {"IP": "灵小甄", "风格": "calm"} - flash_fields = serving._parse_ming_instruction_fields( - SimpleNamespace(instructions="calm", language="粤语", voice="灵小甄") + flash_fields = parse_ming_instruction_fields( + serving, SimpleNamespace(instructions="calm", language="粤语", voice="灵小甄") ) assert flash_fields == {"风格": "calm"} diff --git a/tools/pre_commit/check_tts_adapter.py b/tools/pre_commit/check_tts_adapter.py index d46928662..199c46a30 100644 --- a/tools/pre_commit/check_tts_adapter.py +++ b/tools/pre_commit/check_tts_adapter.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Ratchet on the TTS adapter migration (RFC #4327, #4855). Per-model behaviour is being moved out of ``serving_speech.py`` and into @@ -28,7 +29,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SERVING_SPEECH = Path("vllm_omni/entrypoints/openai/serving_speech.py") # Budgets. These may only go DOWN. See the module docstring. -MAX_MODEL_TYPE_BRANCHES = 9 +MAX_MODEL_TYPE_BRANCHES = 6 _GUIDANCE = """ Move the per-model behaviour into its adapter under diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 813d916fe..50f459a58 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -30,11 +30,9 @@ from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, RequestResponseMetadata, ) -from vllm.inputs import tokens_input from vllm.logger import init_logger from vllm.multimodal.media import MediaConnector from vllm.utils import random_uuid -from vllm.utils.async_utils import make_async from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler @@ -63,17 +61,6 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( tts_entry_stage_archs, ) from vllm_omni.entrypoints.utils import coerce_param_message_types -from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( - build_fish_text_only_prompt_ids, - estimate_fish_voice_clone_prompt_len_from_normalized, - normalize_fish_voice_clone_texts, -) -from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import ( - DEFAULT_PROMPT as MING_DEFAULT_PROMPT, -) -from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import ( - create_instruction as ming_create_instruction, -) from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.speaker_cache import get_speaker_cache @@ -93,12 +80,6 @@ _SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS" # ``tts_adapters.detect_tts_model_type``. Adding a TTS model must not require an # edit to this module. # -# CosyVoice3 talker expects its reference transcript wrapped in the model -# instruction template; without the delimiter the talker re-speaks the -# reference (issue #4644). Matches the offline example/test and upstream demo. -_COSYVOICE3_PROMPT_DELIMITER = "<|endofprompt|>" -_COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DELIMITER}" - # Audex contract: zero-codec / invalid generations arrive as empty terminal # payloads and must fail the request, never serialize as a successful empty # WAV. Covers both the TTS ("audex") and TTA ("audex_tta") pipelines. @@ -109,12 +90,7 @@ _REF_AUDIO_METADATA_FETCH_ATTEMPTS = 3 _REMOTE_REF_AUDIO_SCHEMES = frozenset({"http", "https", "data"}) _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256 _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES = 256 * 1024 * 1024 -_HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256 -_HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024 -_QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key" _TTS_MAX_INSTRUCTIONS_LENGTH = 500 -_TTS_MAX_NEW_TOKENS_MAX = 4096 -_MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT _DEFAULT_VOICE_NAME = "default" @@ -220,64 +196,6 @@ def _validate_path_within_directory(file_path: Path, directory: Path) -> bool: return False -def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str: - """Stable hash of the real Stage 0 conditioning for the prefix cache. - - The talker's vLLM prompt is placeholder token ids; the real inputs are - rebuilt from text / ref_audio / ref_text into inputs_embeds. vLLM hashes - token ids (folded with cache_salt) for prefix caching, so without a salt - every request collides and a hit could reuse KV from a semantically - different input. Tying the salt to the conditioning keeps a hit safe: - identical conditioning may share the prefix, any difference never does. - - Raw request fields alone are not enough for uploaded voices: the request - only carries the voice *name* (ref_audio/ref_text/task_type are resolved - from stored voice data into ``tts_params``). Delete + re-upload under the - same name leaves every raw field identical, so the resolved conditioning - must also be folded in. ``voice_created_at`` bumps on every (re-)upload, - which uniquely identifies the resolved reference artifact together with - the voice name; the decoded ref_audio array itself need not be hashed. - - Local ``ref_audio`` locators are the same class of gap: the raw request - hashes the path/URI string, so an on-disk rewrite is invisible to the - salt. Callers stash the content-aware key from ``_resolve_ref_audio`` as - ``ref_audio_cache_key`` (and ``ref_audio_2_cache_key`` / emotion keys - when those clips exist) so a file edit cannot reuse KV from the previous - reference. - """ - h = hashlib.sha256() - for part in ( - request.input, - request.task_type, - request.language, - request.voice, - request.ref_text, - request.ref_audio, - request.instructions, - request.x_vector_only_mode, - request.speaker_embedding, - ): - h.update(b"\x00") - if part is not None: - h.update(repr(part).encode("utf-8")) - # Fold resolved conditioning that is auto-derived for uploaded voices and - # absent from the raw request (or that a locator string cannot see). - for key in ( - "voice_created_at", - "task_type", - "speaker", - "ref_text", - "x_vector_only_mode", - "ref_audio_cache_key", - "ref_audio_2_cache_key", - ): - h.update(b"\x00") - value = tts_params.get(key) if tts_params is not None else None - if value is not None: - h.update(repr(value).encode("utf-8")) - return h.hexdigest()[:32] - - class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): _diffusion_mode: bool = False _media_connector: MediaConnector | None = None @@ -306,9 +224,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # must not satisfy a later ICL request that needs ref_code (#5049). self._ref_audio_model_artifact_ready: set[tuple[str, bool]] = set() self._request_ref_audio_artifact_keys: dict[str, tuple[str, bool]] = {} - self._higgs_audio_v3_ref_code_cache: OrderedDict[str, tuple[torch.Tensor, int]] = OrderedDict() - self._higgs_audio_v3_ref_code_cache_bytes = 0 - self._higgs_audio_v3_ref_code_inflight: dict[str, asyncio.Task[torch.Tensor]] = {} self._speaker_cache = get_speaker_cache() self._last_upload_ts = 0 self._upload_lock = asyncio.Lock() @@ -438,16 +353,14 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Find and cache the TTS stage (if any) during initialization self._tts_stage = self._find_tts_stage() self._is_tts = self._tts_stage is not None - self._fish_speech_tokenizer = None - self._covo_audio_tokenizer = None - # Cached per process: the CosyVoice3 Qwen tokenizer + resolved model - # path used for dynamic-token sizing. Without this, every request - # re-ran snapshot_download + reloaded the tokenizer (~100 ms on the - # TTFP critical path) in the CosyVoice3 adapter's sampling override. - self._cosyvoice3_tokenizer = None # Determine TTS model type or None self._tts_model_type = self._detect_tts_model_type() + + # Shared executor for blocking adapter preprocessing. It must exist + # before adapter construction so adapters can create their make_async + # wrappers during their own lifecycle initialization. + self._tts_executor = ThreadPoolExecutor(max_workers=1) # Resolve the per-model serving adapter (RFC #4327), keyed on the # detected model-type. Every dedicated TTS model has an adapter; the # adapter owns request validation, prompt/param building, capability @@ -467,37 +380,14 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): available_speakers = self._get_available_speakers() logger.info("Loaded %d supported speakers: %s", len(available_speakers), sorted(available_speakers)) - # Sub-variant inside the full MOSS-TTS family. We collapse all five - # variants onto the same _tts_model_type="moss_tts" because they share - # the same stage layout, but request validation + param building - # differ per HF repo (voice-clone vs dialogue vs ambient-sound vs - # instruction vs streaming voice-clone). - self._moss_variant = self._detect_moss_variant() if self._tts_model_type == "moss_tts" else None - - # GLM-TTS lazy-cached resources (populated on first GLM-TTS request) - self._glm_tts_text_tokenizer: object | None = None - self._glm_tts_text_frontend: object | None = None - # Cache TTS configuration values (computed once, reused per request) self._max_instructions_length = self._compute_max_instructions_length() self._tts_tokenizer = None - self._voxcpm2_tokenizer = None - self._audex_tokenizer = None - self._audex_tta_rvq = None - self._voxcpm2_split_map: dict[int, list[int]] = {} # Batch configuration self._batch_max_items: int = getattr(self.engine_client, "tts_batch_max_items", 32) - # Shared thread pool executor for blocking TTS preprocessing - # operations. max_workers=1 serializes tokenizer access to avoid - # Rust RefCell "Already borrowed" errors from concurrent use. - self._tts_executor = ThreadPoolExecutor(max_workers=1) - self._build_voxtral_prompt_async = make_async(self._build_voxtral_prompt, executor=self._tts_executor) - self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor) - self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor) - def _get_tts_adapter(self): """Return the per-model serving adapter for the current ``_tts_model_type``. @@ -622,73 +512,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """Get all voice names accepted by the API, including the placeholder default.""" return self._get_available_speakers() | {_DEFAULT_VOICE_NAME} - def _estimate_ref_code_len(self, ref_audio: object) -> int | None: - """Estimate ref_code length from ref_audio waveform without running the codec. - - The codec produces one frame per (output_sample_rate / encode_downsample_rate) - audio samples, so ref_code_len = ceil(duration_seconds * codec_frame_rate). - """ - codec_frame_rate = self._adapter.capabilities.codec_frame_rate - if codec_frame_rate is None: - return None - try: - # ref_audio comes from tts_params as [[wav_array, sr]] or similar nested structure - item = ref_audio - while isinstance(item, list) and item: - if len(item) == 2 and isinstance(item[1], (int, float)): - break - item = item[0] - if isinstance(item, list) and len(item) == 2: - wav, sr = item - elif isinstance(item, tuple) and len(item) == 2: - wav, sr = item - else: - return None - sr = int(sr) - if hasattr(wav, "__len__"): - n_samples = len(wav) - elif hasattr(wav, "shape"): - n_samples = wav.shape[-1] if wav.ndim > 1 else wav.shape[0] - else: - return None - if sr <= 0 or n_samples <= 0: - return None - duration = n_samples / sr - return math.ceil(duration * codec_frame_rate) - except Exception: - return None - - def _estimate_prompt_len(self, tts_params: dict[str, Any]) -> int: - """Estimate prompt length so the placeholder matches model-side embeddings.""" - try: - from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import ( - Qwen3TTSPromptEmbedsBuilder, - ) - - if self._tts_tokenizer is None: - from transformers import AutoTokenizer - - model_name = self.engine_client.model_config.model - self._tts_tokenizer = AutoTokenizer.from_pretrained( - model_name, - trust_remote_code=True, - padding_side="left", - ) - hf_config = self.engine_client.model_config.hf_config - talker_config = hf_config.talker_config - task_type = (tts_params.get("task_type") or ["CustomVoice"])[0] - return Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information( - additional_information=tts_params, - task_type=task_type, - tokenize_prompt=lambda t: self._tts_tokenizer(t, padding=False)["input_ids"], - codec_language_id=getattr(talker_config, "codec_language_id", None), - spk_is_dialect=getattr(talker_config, "spk_is_dialect", None), - estimate_ref_code_len=self._estimate_ref_code_len, - ) - except Exception as e: - logger.warning("Failed to estimate TTS prompt length, using fallback 2048: %s", e) - return 2048 - def _get_usage_text_tokenizer(self): """Return a text tokenizer for counting input-text usage tokens. @@ -784,41 +607,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): _SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER: str(usage.input_token_details.audio_tokens), } - def _estimate_fish_ref_code_len(self, ref_audio: object) -> int | None: - """Estimate Fish Speech semantic token length from raw reference audio.""" - from vllm_omni.model_executor.models.fish_speech.dac_utils import ( - DAC_HOP_LENGTH, - DAC_SAMPLE_RATE, - ) - - if not isinstance(ref_audio, (list, tuple)) or len(ref_audio) != 2: - return None - wav, sr = ref_audio - sr = int(sr) - n_samples = len(wav) - if sr <= 0 or n_samples <= 0: - return None - resampled_len = max(1, math.ceil(n_samples * DAC_SAMPLE_RATE / sr)) - return max(1, math.ceil(resampled_len / DAC_HOP_LENGTH)) - - def _estimate_fish_prompt_len(self, text: str, ref_text: str, ref_audio: object) -> int: - """Estimate Fish Speech clone prompt length without encoding reference audio.""" - try: - from transformers import AutoTokenizer - - if self._fish_speech_tokenizer is None: - model_name = self.engine_client.model_config.model - self._fish_speech_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - tokenizer = self._fish_speech_tokenizer - semantic_len = self._estimate_fish_ref_code_len(ref_audio) - if semantic_len is None: - raise ValueError("Failed to estimate Fish Speech semantic token length") - return estimate_fish_voice_clone_prompt_len_from_normalized(tokenizer, text, ref_text, semantic_len) - except Exception as e: - logger.warning("Failed to estimate Fish Speech prompt length, using fallback 2048: %s", e) - return 2048 - def _voice_created_at(self, voice_lower: str) -> int: """Return the upload timestamp of an uploaded voice, or 0 for built-ins. @@ -828,41 +616,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): info = self.uploaded_speakers.get(voice_lower) return int(info.get("created_at", 0)) if info else 0 - async def _build_voxcpm2_prompt( - self, - request: OpenAICreateSpeechRequest, - *, - uploaded_ref: tuple[np.ndarray, int] | None = None, - ) -> dict[str, Any]: - """Build prefill prompt for VoxCPM2 TTS (`prompt_token_ids` padded to full prefill length). - - ``uploaded_ref`` supplies the audio for uploaded voices (no explicit - ``ref_audio`` in the request) so prefill length includes it. - """ - from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import build_voxcpm2_prompt - - self._voxcpm2_encode("") # lazy-init tokenizer + split_map - ref_audio = None - ref_sr = None - voice_profile = None - if request.ref_audio is not None: - ref_audio, ref_sr, _ = await self._resolve_ref_audio(request.ref_audio) - elif uploaded_ref is not None: - wav_np, ref_sr = uploaded_ref - ref_audio = wav_np.tolist() - elif request.voice is not None: - voice_profile = self._adapter.capabilities.precomputed_speakers.get(request.voice.lower()) - return build_voxcpm2_prompt( - hf_config=self.engine_client.model_config.hf_config, - tokenizer=self._voxcpm2_tokenizer, - split_map=self._voxcpm2_split_map, - text=request.input, - ref_audio=ref_audio, - ref_sr=ref_sr, - ref_text=request.ref_text, - voice_profile=voice_profile, - ) - def _load_uploaded_audio(self, voice_name: str) -> tuple[np.ndarray, int] | None: """Load decoded audio samples + sample rate from an uploaded voice's safetensors.""" voice_name_lower = voice_name.lower() @@ -1330,25 +1083,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "sample_rate is not supported by the current TTS model" return None - def _voxcpm2_encode(self, text: str) -> list[int]: - """Tokenize text for VoxCPM2, splitting multichar Chinese tokens.""" - from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import ( - build_cjk_split_map, - split_multichar_chinese, - ) - - if self._voxcpm2_tokenizer is None: - from transformers import AutoTokenizer - - model_name = self.engine_client.model_config.model - tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - self._voxcpm2_split_map = build_cjk_split_map(tokenizer) - self._voxcpm2_tokenizer = tokenizer - logger.info("VoxCPM2 serving: built multichar split map (%d entries)", len(self._voxcpm2_split_map)) - - ids = self._voxcpm2_tokenizer.encode(text, add_special_tokens=True) - return split_multichar_chinese(ids, self._voxcpm2_split_map) - def _validate_ref_audio_format(self, ref_audio: str) -> str | None: """Validate ref_audio is a supported URI format. Returns error or None.""" if not isinstance(ref_audio, str): @@ -1358,445 +1092,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" return None - def _detect_moss_variant(self) -> str: - """Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS - variant family (tts, ttsd, sound_effect, voice_generator, realtime). - - Detection key is the HF repo path / model_name; matches - ``_try_resolve_omni_model_type`` in entrypoints/utils.py so users get - consistent behaviour no matter how they launched the server (--model - OpenMOSS-Team/MOSS-TTSD-v1.0 vs --deploy-config moss_ttsd.yaml). - """ - try: - name = (self.engine_client.model_config.model or "").lower().replace("-", "").replace("_", "") - except Exception: - name = "" - if "realtime" in name: - return "realtime" - if "local" in name: - return "local" - if "ttsd" in name: - return "ttsd" - if "soundeffect" in name: - return "sound_effect" - if "voicegenerator" in name: - return "voice_generator" - return "tts" - - def _get_moss_processor(self): - """Lazily load the upstream MOSS-TTS processor once per server. - - Cached on ``self._moss_processor_cache``. The processor owns its own - audio_tokenizer (~1.6 B params); we keep it on CPU so it doesn't - compete with the talker (~8 GiB) and codec (~7 GiB) for our 96 GiB - GPU — per-request ref-audio encoding is fast enough on CPU. - """ - cached = getattr(self, "_moss_processor_cache", None) - if cached is not None: - return cached - from transformers import AutoProcessor - - model_id = self.engine_client.model_config.model - proc = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) - if hasattr(proc, "audio_tokenizer"): - proc.audio_tokenizer = proc.audio_tokenizer.to("cpu").eval() - self._moss_processor_cache = proc - return proc - - def _get_moss_ref_encoder(self): - """Lazily build the per-server MOSS reference-audio encoder once.""" - cached = getattr(self, "_moss_ref_encoder", None) - if cached is not None: - return cached - from vllm_omni.model_executor.models.moss_tts.reference_encoder import build_reference_encoder - - # The variant's encode geometry (n_vq, working sample rate) is derived - # inside the model package; this layer only supplies the processor and - # the process-wide speaker cache. - encoder = build_reference_encoder( - self._get_moss_processor(), - variant=self._moss_variant, - speaker_cache=self._speaker_cache, - ) - self._moss_ref_encoder = encoder - return encoder - - async def _build_moss_tts_params( - self, - request: OpenAICreateSpeechRequest, - *, - has_inline_ref_audio: bool = False, - ) -> dict[str, Any]: - """Build the talker prompt + ``additional_information`` payload for any - MOSS-TTS-family request (nano + 5 full variants). - - For the legacy ``moss_tts_nano`` model_type, keeps the original nano - contract (``{text, mode=voice_clone, prompt_audio_array}``); the - caller still uses a ``[1]`` placeholder prompt. - - For the full MOSS-TTS family (``MossTTSDelay*`` / ``MossTTSRealtime*``) - we **call the upstream processor server-side** to produce the unified - ``(text_ids, audio_codes)`` shape the talker actually consumes — same - flow as ``examples/.../moss_tts/end2end.py:_build_unified_codes``. - Returns ``{prompt_token_ids: list[int], codes.ref: torch.LongTensor, - max_new_frames, ...}``. The caller treats ``prompt_token_ids`` as the - prompt and forwards the rest as ``additional_information``. - """ - import torch # local to avoid pulling torch at module import time - - v = self._moss_variant - - # ---- Legacy nano path (unchanged) ---- - if v is None: # moss_tts_nano - params: dict[str, Any] = { - "text": [request.input or ""], - "mode": ["voice_clone"], - } - if request.max_new_tokens is not None: - params["max_new_frames"] = [request.max_new_tokens] - wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) - params["prompt_audio_array"] = [[wav_list, sr]] - params["ref_audio_cache_key"] = cache_key - return params - - # ---- MOSS-TTS-Realtime: keep the old prompt_audio_array path ---- - # ``AutoProcessor.from_pretrained`` doesn't auto-discover - # ``MossTTSRealtimeProcessor`` (no ``processor_config.json`` in the - # snapshot), and Realtime's prompt format diverges from MossTTSDelay - # (16-channel grid, separate per-step text feed). The - # ``prompt_audio_array`` shape lines up well enough with what the - # talker reads for short prompts; full Realtime support needs a - # separate processor.from_module path which we don't wire here. - if v == "realtime": - params: dict[str, Any] = { - "text": [request.input or ""], - "mode": ["voice_clone"], - } - if request.max_new_tokens is not None: - params["max_new_frames"] = [request.max_new_tokens] - wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) - params["prompt_audio_array"] = [[wav_list, sr]] - params["ref_audio_cache_key"] = cache_key - return params - - # ---- MossTTSDelay family (tts/ttsd/sound_effect/voice_generator) - # and MOSS-TTS-Local-Transformer-v1.5: call the upstream processor - # server-side to produce unified codes. Local-v1.5 ships its own - # AutoProcessor (processor_config.json + processing_moss_tts.py) and - # reuses this exact build_user_message/encode_audios_from_wav path in - # the offline example (examples/.../moss_tts/end2end.py: - # _build_unified_codes) -- it is NOT in the same boat as Realtime - # (no processor_config.json there), so it must not fall back to the - # prompt_audio_array path above (which the talker's preprocess() - # never reads -- info_dict["codes"]["ref"] is the only thing it - # consumes, so skipping this path silently drops all voice-clone - # conditioning and produces unconditioned/garbage audio online). ---- - proc = self._get_moss_processor() - - user_kwargs: dict[str, Any] = {"text": request.input or ""} - resolve_keys: dict[int, str] = {} - if v in ("tts", "local", "ttsd"): - user_kwargs["reference"], resolve_keys = await self._encode_moss_references( - request, - has_inline_ref_audio=has_inline_ref_audio, - two_speaker=(v == "ttsd"), - ) - elif v == "sound_effect": - user_kwargs["text"] = request.input or "" # may be empty - user_kwargs["ambient_sound"] = request.ambient_sound or "" - if request.duration_seconds is not None: - user_kwargs["tokens"] = max(1, int(float(request.duration_seconds) * 12.5)) - elif request.max_new_tokens is not None: - user_kwargs["tokens"] = int(request.max_new_tokens) - elif v == "voice_generator": - user_kwargs["instruction"] = request.instructions or "" - - # Optional language tag for the spoken-text variants. MOSS-TTS-v1.5's - # headline improvement is multilingual synthesis when the language is - # given (build_user_message(..., language=...)); 1.0 ignores it - # gracefully. Sound-effect output is non-verbal, so skip it there. - if v in ("tts", "local", "ttsd", "voice_generator") and getattr(request, "language", None): - user_kwargs["language"] = request.language - - # Build the unified-codes prompt: (L, 1+n_vq) where col 0 is text/special - # tokens and cols 1..n_vq are the delay-pattern audio code grid (mostly - # audio_pad_code outside the reference block). - user_msg = proc.build_user_message(**user_kwargs) - batch = proc(conversations=[[user_msg]], mode="generation") - unified = batch["input_ids"][0] # torch.LongTensor (L, 1+n_vq) - text_ids: list[int] = unified[:, 0].tolist() - audio_codes: torch.Tensor = unified[:, 1:].contiguous().to(torch.int64) - - params: dict[str, Any] = { - "prompt_token_ids": text_ids, - "codes": {"ref": audio_codes}, - } - if request.max_new_tokens is not None: - params["max_new_frames"] = [request.max_new_tokens] - if 0 in resolve_keys: - params["ref_audio_cache_key"] = resolve_keys[0] - if 1 in resolve_keys: - params["ref_audio_2_cache_key"] = resolve_keys[1] - return params - - async def _encode_moss_references( - self, - request: OpenAICreateSpeechRequest, - *, - has_inline_ref_audio: bool, - two_speaker: bool, - ) -> tuple[list, dict[int, str]]: - """Encode the request's reference clip(s) into MOSS RVQ code tensors. - - Reference encoding + speaker caching lives in the model package - (moss_tts.reference_encoder), mirroring Fish Speech / CosyVoice3 / - Qwen3-TTS which keep reference handling with the model rather than in - this shared serving file. This method only resolves the serving-side - inputs the model helper needs: the generic audio resolver/artifact-key - lookups and whether the request may use the named-voice cache. - - Returns ``(codes_per_speaker, resolve_keys)`` where ``resolve_keys`` - maps the reference slot (0 = ref_audio, 1 = ref_audio_2) to its - content-aware resolve key, for salting the KV prefix cache. - """ - from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_request_references - - # Named-voice caching is only valid for uploaded speakers without an - # inline ref_audio: ``request.voice`` plus a file/URL would otherwise - # key on (name, created_at=0) and skip the content-aware resolve. - raw_voice = getattr(request, "voice", None) - raw_voice = raw_voice.strip() if isinstance(raw_voice, str) else "" - voice_lower = raw_voice.lower() - use_named_voice = bool(voice_lower) and voice_lower in self.uploaded_speakers and not has_inline_ref_audio - voice = voice_lower if use_named_voice else "" - voice_created = self._voice_created_at(voice) if voice else 0 - - return await encode_request_references( - self._get_moss_ref_encoder(), - request.ref_audio, - request.ref_audio_2 if two_speaker else None, - resolve_ref_audio=self._resolve_ref_audio, - get_artifact_key=self._get_resolved_ref_audio_artifact_key, - voice_name=voice or None, - voice_created_at=voice_created, - ) - - async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest): - """Build prompt_token_ids for higgs_audio_v2 via the upstream processor. - - Plain-text path: runs ``build_plain_text_prompt`` and returns the - token-only prompt. Voice-clone path (``ref_audio`` + ``ref_text``): - resolves the reference clip via ``_resolve_ref_audio``, runs - ``build_voice_clone_prompt`` (which encodes the clip through HF's - ``HiggsAudioV2TokenizerModel`` loaded from the k2-fsa/OmniVoice - ``audio_tokenizer/`` subdirectory), and attaches the encoded - ``audio_input_ids`` + ``audio_input_ids_mask`` tensors via - ``additional_information`` so the talker substitutes them at the - prompt-side audio placeholders. - """ - from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( - build_plain_text_prompt, - build_voice_clone_prompt, - input_ids_to_python_list, - ) - - processor = await self._resolve_higgs_audio_v2_processor() - - if request.ref_audio is None: - inputs = await asyncio.to_thread(build_plain_text_prompt, processor, request.input) - prompt_token_ids = input_ids_to_python_list(inputs) - return tokens_input(prompt_token_ids=prompt_token_ids) - - wav_list, sr, _ = await self._resolve_ref_audio(request.ref_audio) - wav = np.asarray(wav_list, dtype=np.float32) - out = await asyncio.to_thread( - build_voice_clone_prompt, - processor, - request.input, - wav, - int(sr), - request.ref_text or "", - ) - prompt = tokens_input(prompt_token_ids=out["prompt_token_ids"]) - # Pass tensors at the top level of additional_information (NOT list- - # wrapped). ``vllm_omni.data_entry_keys.serialize_payload`` routes - # bare ``torch.Tensor`` values through ``_serialize_tensor``; a list - # containing tensors would fall into the ``list_data`` field which - # msgspec cannot serialize and the tensors would be dropped over the - # process boundary (silent voice-clone failure). - prompt["additional_information"] = { - "audio_input_ids": out["audio_input_ids"], - "audio_input_ids_mask": out["audio_input_ids_mask"], - } - return prompt - - async def _resolve_higgs_audio_v2_processor(self): - """Lazy-load the AutoProcessor for higgs_audio_v2 (once per serving instance).""" - cached = getattr(self, "_higgs_audio_v2_processor", None) - if cached is not None: - return cached - - from transformers import AutoProcessor - - model_path = None - for stage in self.engine_client.stage_configs: - model_path = getattr(getattr(stage, "engine_args", None), "model", None) - if model_path: - break - if model_path is None: - # Fallback: the orchestrator stores the served model id on the engine - # itself (set by AsyncOmniEngine.__init__). Stage-level engine_args - # may not surface ``model`` when the deploy yaml doesn't set it per - # stage (the CLI-passed model id is the single source of truth). - model_path = getattr(self.engine_client, "model", None) - if model_path is None: - raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs") - processor = AutoProcessor.from_pretrained(model_path) - self._higgs_audio_v2_processor = processor - return processor - - # ---- higgs-audio v3 ---- - - async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest): - """Build prompt_token_ids for higgs_audio_v3. - - Plain-text path: builds ``[tts, text, tokens, audio]``. - Voice-clone path: encodes reference audio, applies delay pattern, and - records reference-audio prompt positions while submitting valid token IDs. - """ - adapter = await self._resolve_higgs_audio_v3_adapter() - - if request.ref_audio is None: - prompt_ids = adapter.build_prompt(request.input) - return tokens_input(prompt_token_ids=prompt_ids) - - # Voice clone - from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import ( - apply_delay_pattern, - encode_reference_audio, - ) - - wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) - artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key) - wav = np.asarray(wav_list, dtype=np.float32) - ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes( - artifact_key, - wav, - int(sr), - encode_reference_audio, - apply_delay_pattern, - ) - del cache_hit, inflight_wait - - prompt_ids = adapter.build_prompt( - request.input, - num_ref_tokens=int(ref_codes_delayed.shape[0]), - reference_text=request.ref_text or None, - ) - prompt_ids, audio_placeholder_positions = adapter.prepare_prompt_for_engine(prompt_ids) - prompt = tokens_input(prompt_token_ids=prompt_ids) - import torch - - prompt["additional_information"] = { - "audio_input_ids": ref_codes_delayed.to(torch.long), - "audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool), - "audio_placeholder_positions": audio_placeholder_positions, - "ref_audio_cache_key": cache_key, - } - # Placeholder prompt ids do not change when a same-path file is - # rewritten at the same duration; fold the content-aware resolve key - # into cache_salt so prefix caching cannot reuse the previous KV. - prompt["cache_salt"] = _conditioning_cache_salt(request, prompt["additional_information"]) - return prompt - - async def _resolve_higgs_audio_v3_ref_codes( - self, - artifact_key: str | None, - wav: np.ndarray, - sr: int, - encode_reference_audio, - apply_delay_pattern, - ) -> tuple[torch.Tensor, bool, bool]: - ref_codes_delayed = self._get_higgs_audio_v3_ref_codes(artifact_key) - if ref_codes_delayed is not None: - return ref_codes_delayed, True, False - if not artifact_key: - ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr) - return apply_delay_pattern(ref_codes_raw), False, False - - task = self._higgs_audio_v3_ref_code_inflight.get(artifact_key) - if task is not None: - return (await task).clone(), False, True - - async def _encode_and_cache() -> torch.Tensor: - ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr) - delayed = apply_delay_pattern(ref_codes_raw) - self._put_higgs_audio_v3_ref_codes(artifact_key, delayed) - cached = self._get_higgs_audio_v3_ref_codes(artifact_key) - return cached if cached is not None else delayed.detach().to("cpu", dtype=torch.long).contiguous() - - task = asyncio.create_task(_encode_and_cache()) - self._higgs_audio_v3_ref_code_inflight[artifact_key] = task - try: - return (await task).clone(), False, False - finally: - if self._higgs_audio_v3_ref_code_inflight.get(artifact_key) is task: - self._higgs_audio_v3_ref_code_inflight.pop(artifact_key, None) - - def _get_higgs_audio_v3_ref_codes(self, artifact_key: str | None) -> torch.Tensor | None: - if not artifact_key: - return None - cached = self._higgs_audio_v3_ref_code_cache.get(artifact_key) - if cached is None: - return None - self._higgs_audio_v3_ref_code_cache.move_to_end(artifact_key) - return cached[0].clone() - - def _put_higgs_audio_v3_ref_codes(self, artifact_key: str, codes: torch.Tensor) -> None: - if _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES <= 0 or _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES <= 0 or not artifact_key: - return - cached_codes = codes.detach().to("cpu", dtype=torch.long).contiguous() - size = int(cached_codes.numel() * cached_codes.element_size()) - if size > _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES: - return - previous = self._higgs_audio_v3_ref_code_cache.pop(artifact_key, None) - if previous is not None: - self._higgs_audio_v3_ref_code_cache_bytes -= previous[1] - self._higgs_audio_v3_ref_code_cache[artifact_key] = (cached_codes, size) - self._higgs_audio_v3_ref_code_cache_bytes += size - while len(self._higgs_audio_v3_ref_code_cache) > _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES: - _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False) - self._higgs_audio_v3_ref_code_cache_bytes -= old_size - while self._higgs_audio_v3_ref_code_cache_bytes > _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES: - _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False) - self._higgs_audio_v3_ref_code_cache_bytes -= old_size - - async def _resolve_higgs_audio_v3_adapter(self): - """Lazy-load the tokenizer adapter for higgs_audio_v3.""" - cached = getattr(self, "_higgs_audio_v3_adapter", None) - if cached is not None: - return cached - - from transformers import AutoTokenizer - - from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import ( - HiggsAudioV3TokenizerAdapter, - ) - - model_path = None - for stage in self.engine_client.stage_configs: - model_path = getattr(getattr(stage, "engine_args", None), "model", None) - if model_path: - break - if model_path is None: - model_path = getattr(self.engine_client, "model", None) - if model_path is None: - raise RuntimeError("higgs_audio_v3 serving could not resolve model path") - tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) - adapter = HiggsAudioV3TokenizerAdapter(tokenizer) - self._higgs_audio_v3_adapter = adapter - return adapter - @staticmethod def _local_ref_audio_stat_path(ref_audio_str: str) -> str | None: """Filesystem path to stat for a local locator, else ``None``. @@ -2043,14 +1338,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _tts_x_vector_only(tts_params: dict[str, Any]) -> bool: return bool((tts_params.get("x_vector_only_mode") or [False])[0]) - def _qwen3_tts_can_use_ref_audio_artifact_only(self, tts_params: dict[str, Any], artifact_key: str | None) -> bool: - if self._tts_model_type != "qwen3_tts": - return False - x_vector_only = self._tts_x_vector_only(tts_params) - if not artifact_key or (artifact_key, x_vector_only) not in self._ref_audio_model_artifact_ready: - return False - return (tts_params.get("task_type") or ["CustomVoice"])[0] == "Base" - def _track_ref_audio_artifact_warmup( self, request_id: str, artifact_key: str | None, x_vector_only: bool = False ) -> None: @@ -2075,145 +1362,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): resolved.append((wav_list, sr)) return resolved - # ---- Ming TTS helpers ---- - - def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate): - from torchaudio.functional import resample as resample_audio - - from vllm_omni.model_executor.models.ming_tts.config_ming_tts import SAMPLE_RATE - - waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1) - if int(sample_rate) != SAMPLE_RATE: - waveform = resample_audio(waveform, int(sample_rate), SAMPLE_RATE) - return waveform - - def _build_ming_reference_waveforms( - self, - ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None, - ) -> tuple[list[torch.Tensor], list[int]]: - if ref_audio_data is None: - return [], [] - items = ref_audio_data if isinstance(ref_audio_data, list) else [ref_audio_data] - waveforms = [torch.as_tensor(samples, dtype=torch.float32).reshape(1, -1) for samples, _ in items] - sample_rates = [int(sample_rate) for _, sample_rate in items] - if any(sample_rate <= 0 for sample_rate in sample_rates): - raise ValueError(f"Ming reference audio sample rates must be positive, got {sample_rates}") - return waveforms, sample_rates - - def _parse_ming_instruction_fields( - self, - request, - *, - include_voice=False, - plain_text_passthrough=False, - ): - instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None - instruction_dict: dict[str, Any] = {} - - voice_lower = request.voice.lower() if isinstance(request.voice, str) else None - if include_voice and request.voice and not (voice_lower and voice_lower in self.uploaded_speakers): - instruction_dict["IP"] = request.voice - - if instruction_text: - try: - parsed = json.loads(instruction_text) - except json.JSONDecodeError: - parsed = None - if isinstance(parsed, dict): - instruction_dict.update(parsed) - elif instruction_dict or not plain_text_passthrough: - instruction_dict["风格"] = instruction_text - else: - return instruction_text - - return instruction_dict or None - - def _parse_ming_instruction(self, request: OpenAICreateSpeechRequest) -> Any: - """Build a Ming instruction payload from OpenAI speech fields.""" - return self._parse_ming_instruction_fields( - request, - include_voice=True, - plain_text_passthrough=True, - ) - - def _build_ming_dense_prompt( - self, - request: OpenAICreateSpeechRequest, - *, - ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None = None, - voice_name: str | None = None, - voice_created_at: int = 0, - ) -> dict[str, Any]: - """Build a Ming dense prompt directly from the OpenAI speech request.""" - from transformers import AutoTokenizer - - from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( - KEY_MAX_DECODE_STEPS, - KEY_SPEAKER_SAMPLE_RATES, - KEY_SPEAKER_WAVEFORM, - KEY_SPEAKER_WAVEFORM_LENGTHS, - ) - from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt - - if self._tts_tokenizer is None: - model_name = self.engine_client.model_config.model - trust_remote_code = bool(getattr(self.engine_client.model_config, "trust_remote_code", False)) - self._tts_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code) - - ref_text = request.ref_text - reference_waveforms, reference_sample_rates = self._build_ming_reference_waveforms(ref_audio_data) - reference_waveform = torch.cat(reference_waveforms, dim=-1) if reference_waveforms else None - prompt_waveforms = ( - [ - self._coerce_ming_prompt_waveform(waveform, sample_rate) - for waveform, sample_rate in zip(reference_waveforms, reference_sample_rates, strict=True) - ] - if ref_text is not None - else [] - ) - prompt_waveform = torch.cat(prompt_waveforms, dim=-1) if prompt_waveforms else None - speaker_embedding = request.speaker_embedding - pending_speaker_extraction = speaker_embedding is None and reference_waveform is not None - use_zero_spk_emb = not pending_speaker_extraction and prompt_waveform is None and speaker_embedding is None - - runtime_controls = {} - if request.max_new_tokens is not None: - runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens - - prompt_dict = build_ming_dense_prompt( - self._tts_tokenizer, - # bgm / music-prompt mode not supported online; - # requires prompt_mode API extension (deferred). - prompt=_MING_DEFAULT_PROMPT, - text=request.input, - runtime_controls=runtime_controls or None, - instruction=self._parse_ming_instruction(request), - prompt_text=ref_text, - prompt_waveform=prompt_waveform, - speaker_embedding=speaker_embedding, - speaker_count=len(reference_waveforms) if pending_speaker_extraction else None, - use_zero_spk_emb=use_zero_spk_emb, - ) - additional_information = prompt_dict["additional_information"] - if pending_speaker_extraction: - additional_information[KEY_SPEAKER_WAVEFORM] = reference_waveform - additional_information[KEY_SPEAKER_WAVEFORM_LENGTHS] = torch.tensor( - [int(waveform.shape[-1]) for waveform in reference_waveforms], - dtype=torch.int32, - ) - additional_information[KEY_SPEAKER_SAMPLE_RATES] = torch.tensor( - reference_sample_rates, - dtype=torch.int32, - ) - if voice_name: - additional_information["voice_name"] = voice_name - additional_information["voice_created_at"] = int(voice_created_at) - prompt = tokens_input(prompt_token_ids=prompt_dict["prompt_token_ids"]) - prompt["prompt"] = prompt_dict["prompt"] - prompt["text"] = prompt_dict["text"] - prompt["additional_information"] = prompt_dict["additional_information"] - return prompt - async def _generate_audio_chunks( self, generator, @@ -2542,534 +1690,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): key = "audio" if "audio" in mm else ("model_outputs" if "model_outputs" in mm else None) return mm, key - def _build_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: - """Build TTS parameters from request. - - Processes each parameter if present, skips if not. - Values are wrapped in lists as required by the model. - """ - params: dict[str, Any] = {} - - # Text content (always required) - params["text"] = [request.input] - - # Task type - if request.task_type is not None: - params["task_type"] = [request.task_type] - else: - params["task_type"] = ["CustomVoice"] - - # Language - if request.language is not None: - params["language"] = [request.language] - else: - params["language"] = ["Auto"] - - # Speaker (voice) - if request.voice is not None: - voice_lower = request.voice.lower() - precomputed_speakers = self._adapter.capabilities.precomputed_speakers if self._adapter is not None else {} - params["speaker"] = [request.voice] - params["voice_created_at"] = [self._voice_created_at(voice_lower)] - - # Uploaded voices use task_type="Base" (CustomVoice requires built-in spk_id). - # If ref_text was provided at upload time, use in-context cloning; otherwise x_vector only. - if voice_lower in self.uploaded_speakers and request.ref_audio is None: - speaker_info = self.uploaded_speakers[voice_lower] - - # Check if this voice was uploaded with a pre-computed embedding. - # Populate request.speaker_embedding so the existing code path - # (below) handles voice_clone_prompt and x_vector_only_mode. - embedding = self._get_uploaded_speaker_embedding(request.voice) - if embedding is not None: - request.speaker_embedding = embedding - params["speaker"] = [voice_lower] - params["task_type"] = ["Base"] - logger.info("Auto-set speaker_embedding for uploaded voice: %s", request.voice) - else: - audio_data = self._get_uploaded_audio_data(request.voice) - if not audio_data: - raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted") - stored_ref_text = speaker_info.get("ref_text") - params["speaker"] = [voice_lower] - params["ref_audio"] = [audio_data] - params["task_type"] = ["Base"] - if stored_ref_text: - params["ref_text"] = [stored_ref_text] - params["x_vector_only_mode"] = [False] - else: - params["x_vector_only_mode"] = [True] - logger.info( - "Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text) - ) - elif voice_lower in precomputed_speakers and request.ref_audio is None: - profile = precomputed_speakers[voice_lower] - mode = str(profile.get("mode") or "xvec").lower() - params["speaker"] = [voice_lower] - params["task_type"] = ["Base"] - params["x_vector_only_mode"] = [mode != "icl"] - ref_text = request.ref_text or profile.get("ref_text") - if isinstance(ref_text, str) and ref_text.strip(): - params["ref_text"] = [ref_text] - ref_code_length = profile.get("ref_code_length") - if mode == "icl" and ref_code_length: - params["ref_code_length"] = [int(ref_code_length)] - logger.info("Using precomputed Qwen3-TTS custom voice profile: %s (mode=%s)", voice_lower, mode) - - elif params["task_type"][0] == "CustomVoice": - params["speaker"] = ["Vivian"] # Default for CustomVoice - - # Instructions for style/emotion control - if request.instructions is not None: - params["instruct"] = [request.instructions] - else: - params["instruct"] = [""] - - # Voice clone: ref_audio resolved in create_speech(), not here. - if request.ref_text is not None: - params["ref_text"] = [request.ref_text] - if request.speaker_embedding is not None: - # Store as plain float list (not tensor) so it survives msgspec - # serialization through the EngineCore IPC boundary. The talker's - # _build_prompt_embeds converts it back to a tensor on the GPU. - params["voice_clone_prompt"] = [ - { - "ref_spk_embedding": list(request.speaker_embedding), - } - ] - # speaker_embedding implies x_vector_only_mode - params["x_vector_only_mode"] = [True] - elif request.x_vector_only_mode is not None: - params["x_vector_only_mode"] = [request.x_vector_only_mode] - - # Generation parameters - if request.max_new_tokens is not None: - params["max_new_tokens"] = [request.max_new_tokens] - else: - params["max_new_tokens"] = [2048] - - if request.initial_codec_chunk_frames is not None: - params["initial_codec_chunk_frames"] = [request.initial_codec_chunk_frames] - - if request.non_streaming_mode is not None: - params["non_streaming_mode"] = [request.non_streaming_mode] - # Preserve the legacy VoiceDesign fallback when the request omits an - # explicit override. CustomVoice and Base rely on model defaults - # (True and False respectively). - elif params["task_type"][0] == "VoiceDesign": - params["non_streaming_mode"] = [True] - - return params - - # ---- Voxtral TTS helpers ---- - - def _build_voxtral_prompt(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: - """Build Voxtral TTS engine prompt, supporting both preset voices and inline - ``ref_audio`` (base64 or data URI).""" - from mistral_common.protocol.speech.request import SpeechRequest - - text = request.input - voice = request.voice - ref_audio = request.ref_audio - if not voice and not ref_audio: - raise ValueError("Voxtral requires either a voice name or ref_audio.") - # mistral_common expects raw base64 (no data: prefix) - if ref_audio is not None and isinstance(ref_audio, str) and ref_audio.startswith("data:"): - _, _, ref_audio = ref_audio.partition(",") - if self._tts_tokenizer is None: - from vllm.tokenizers import cached_tokenizer_from_config - - mistral_tokenizer = cached_tokenizer_from_config(self.engine_client.model_config) - self._tts_tokenizer = mistral_tokenizer.instruct - if voice is not None: - tokens = self._tts_tokenizer.encode_speech_request(SpeechRequest(input=text, voice=voice)).tokens - prompt = tokens_input(prompt_token_ids=tokens) - prompt["additional_information"] = {"voice": [voice]} - return prompt - else: - tokenized = self._tts_tokenizer.encode_speech_request(SpeechRequest(input=text, ref_audio=ref_audio)) - audio = tokenized.audios[0] - return { - "prompt_token_ids": tokenized.tokens, - "multi_modal_data": {"audio": [(audio.audio_array, audio.sampling_rate)]}, - } - - # ---- Step-Audio2 helpers ---- - - def _build_step_audio2_prompt( - self, - request: OpenAICreateSpeechRequest, - ) -> dict[str, Any]: - """Build prompt for Step-Audio2 TTS. - - Constructs the chat prompt with ```` as the last token - of the assistant turn (without ``<|im_end|>``), so the thinker - continues generating audio tokens. - - Prompt format:: - <|im_start|>system\\n{system_prompt}<|im_end|>\\n - <|im_start|>user\\n{input_text}<|im_end|>\\n - <|im_start|>assistant\\n - """ - system_prompt = getattr(request, "instructions", None) or "You are a voice assistant. Read the text aloud." - text = request.input - - raw_prompt = ( - f"<|im_start|>system\n{system_prompt}<|im_end|>\n" - f"<|im_start|>user\n{text}<|im_end|>\n" - f"<|im_start|>assistant\n" - ) - return {"prompt": raw_prompt} - - # ---- Fish Speech helpers ---- - - def _build_fish_speech_prompt( - self, - request: OpenAICreateSpeechRequest, - ref_audio_data: tuple[list[float], int] | None = None, - *, - has_inline_ref_audio: bool = False, - ) -> dict[str, Any]: - """Build prompt for Fish Speech S2 Pro. - - Without voice cloning: - <|im_start|>system\\nconvert the provided text to speech<|im_end|> - <|im_start|>user\\n{text}<|im_end|>\\n<|im_start|>assistant\\n<|voice|> - - With voice cloning (ref_audio + ref_text): - <|im_start|>system\\nconvert the provided text to speech reference to the following... - <|im_end|>\\n<|im_start|>user\\n{text}<|im_end|>\\n<|im_start|>assistant\\n<|voice|> - """ - from transformers import AutoTokenizer - - if self._fish_speech_tokenizer is None: - model_name = self.engine_client.model_config.model - self._fish_speech_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - tokenizer = self._fish_speech_tokenizer - - if ref_audio_data is None or not request.ref_text: - prompt_ids, normalized_text = build_fish_text_only_prompt_ids(tokenizer, request.input) - - # Keep the prompt-dict metadata shape aligned with the existing text-only - # TTS entrypoints: scalar values are wrapped in single-item lists before - # EngineCore serialization. Structured clone below is different because - # model-side preprocess consumes concrete per-request scalar fields. - additional_information: dict[str, Any] = { - "text": [normalized_text], - } - if request.max_new_tokens is not None: - additional_information["max_new_tokens"] = [request.max_new_tokens] - prompt = tokens_input(prompt_token_ids=prompt_ids) - prompt["additional_information"] = additional_information - return prompt - - wav_samples, sr = ref_audio_data - normalized_text, normalized_ref_text = normalize_fish_voice_clone_texts(request.input, request.ref_text) - ph_len = self._estimate_fish_prompt_len(normalized_text, normalized_ref_text, ref_audio_data) - - # Structured clone: scalars (not list-wrapped) because model-side - # preprocess() consumes per-request fields directly. - additional_information: dict[str, Any] = { - "text": normalized_text, - "ref_text": normalized_ref_text, - "ref_audio_wav": torch.from_numpy(np.asarray(wav_samples, dtype=np.float32)), - "ref_audio_sr": int(sr), - "fish_structured_voice_clone": True, - } - # Pass voice identity for model-side DAC code caching. - if request.voice is not None: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: - additional_information["voice_name"] = voice_lower - additional_information["voice_created_at"] = self._voice_created_at(voice_lower) - if request.max_new_tokens is not None: - additional_information["max_new_tokens"] = request.max_new_tokens - prompt = tokens_input(prompt_token_ids=[1] * ph_len) - prompt["additional_information"] = additional_information - return prompt - - # ---- CosyVoice3 helpers ---- - - async def _build_cosyvoice3_prompt( - self, - request: OpenAICreateSpeechRequest, - *, - has_inline_ref_audio: bool = False, - ) -> dict[str, Any]: - """Build prompt for CosyVoice3. - - CosyVoice3 uses multimodal input with reference audio for voice cloning. - The prompt format matches the offline example: text prompt + audio data - + mm_processor_kwargs with prompt_text. - """ - # Resolve reference audio - wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio) - audio_data = (np.asarray(wav_samples, dtype=np.float32), sr) - - # Wrap the reference transcript in the CosyVoice3 instruction template - # so the talker emits target-only speech (see _COSYVOICE3_PROMPT_PREFIX). - # Skip if the caller already supplied a formatted prompt_text. - ref_text = request.ref_text or "" - if _COSYVOICE3_PROMPT_DELIMITER not in ref_text: - ref_text = f"{_COSYVOICE3_PROMPT_PREFIX}{ref_text}" - mm_kwargs: dict[str, Any] = { - "prompt_text": ref_text, - "sample_rate": sr, - } - # Pass voice metadata for caching in the processor - if request.voice: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: - mm_kwargs["voice_name"] = voice_lower - mm_kwargs["voice_created_at"] = self._voice_created_at(voice_lower) - - return { - "prompt": request.input, - "multi_modal_data": { - "audio": audio_data, - }, - "mm_processor_kwargs": mm_kwargs, - } - - # ---- Covo-Audio helpers ---- - - def _build_covo_audio_prompt( - self, - request: OpenAICreateSpeechRequest, - ) -> dict[str, Any]: - """Build a chat-style prompt for Covo-Audio-Chat. - - Covo-Audio requires a specific system prompt that instructs the model - to interleave text and audio tokens in its output. We render the - messages through the chat template and pass prompt_token_ids so that - the engine does not need to re-tokenize. - """ - from transformers import AutoTokenizer - - from vllm_omni.model_executor.models.covo_audio.prompt_utils import ( - build_covo_audio_prompt_token_ids, - ) - - if self._covo_audio_tokenizer is None: - model_name = self.engine_client.model_config.model - try: - self._covo_audio_tokenizer = AutoTokenizer.from_pretrained( - model_name, - trust_remote_code=True, - ) - except Exception as exc: - raise RuntimeError(f"Failed to load Covo-Audio tokenizer from '{model_name}': {exc}") from exc - - prompt_ids = build_covo_audio_prompt_token_ids( - self._covo_audio_tokenizer, - request.input, - ) - return {"prompt_token_ids": prompt_ids} - - def _get_audex_tokenizer(self): - """Load (once per process) the Audex thinker tokenizer for CFG/TTA prompts. - - The folder follows the serving stage: the full-pipeline thinker - (``audex_omni``) tokenizes with ``checkpoint_folder_full``; the TTS - and TTA thinkers use ``checkpoint_folder_audiogen``. - """ - if self._audex_tokenizer is None: - from transformers import AutoTokenizer - - from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot - - stage = getattr(getattr(self._tts_stage, "engine_args", None), "model_stage", None) - if stage == "audex_omni": - profile, folder = "full", "checkpoint_folder_full" - elif stage == "audex_tta_thinker": - profile, folder = "tta", "checkpoint_folder_audiogen" - else: - profile, folder = "tts", "checkpoint_folder_audiogen" - # The serving model_config may already hold the RESOLVED per-stage - # checkpoint folder (stage init joins model_subdir onto the - # snapshot root); joining again would produce a non-existent path - # that transformers then mistakes for a repo id. - model_path = os.path.normpath(self.engine_client.model_config.model) - if os.path.basename(model_path).startswith("checkpoint_folder"): - root = os.path.dirname(model_path) - else: - root = ensure_audex_snapshot(model_path, profile=profile) - self._audex_tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, folder)) - return self._audex_tokenizer - - # ---- GLM-TTS helpers ---- - - async def _build_glm_tts_prompt( - self, - request: OpenAICreateSpeechRequest, - *, - has_inline_ref_audio: bool = False, - ) -> dict[str, Any]: - """Build prompt for GLM-TTS. - - Uses the multimodal processor path (same as CosyVoice3): - - prompt: synthesis text - - multi_modal_data["audio"]: (wav_samples, sr) reference audio - - mm_processor_kwargs["prompt_text"]: reference text transcript - - AR preprocess() builds [PromptText | Text | BOA | PromptSpeechTokens + ATS]. - DiT receives prompt_token, prompt_feat, embedding for conditioning. - """ - # Voice cloning requires ref_audio + ref_text - if request.ref_audio is not None and request.ref_text: - wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio) - audio_data = (np.asarray(wav_samples, dtype=np.float32), int(sr)) - - mm_kwargs: dict[str, Any] = { - "prompt_text": request.ref_text, - } - if request.voice: - voice_lower = request.voice.lower() - if voice_lower in self.uploaded_speakers and not has_inline_ref_audio: - mm_kwargs["voice_name"] = voice_lower - mm_kwargs["voice_created_at"] = self._voice_created_at(voice_lower) - - return { - "prompt": request.input, - "multi_modal_data": { - "audio": audio_data, - }, - "mm_processor_kwargs": mm_kwargs, - "additional_information": self._build_glm_tts_prefill_metadata( - request.input, - request.ref_text, - ), - } - - raise ValueError("GLM-TTS requires ref_audio and ref_text for voice cloning.") - - def _glm_tts_text_tokenizer_and_frontend(self): - from vllm_omni.model_executor.models.glm_tts.glm_tts import ( - load_glm_tts_tokenizer, - resolve_glm_tts_tokenizer_path, - ) - from vllm_omni.model_executor.models.glm_tts.text_frontend import GLMTTSTextFrontend - - cached = self._glm_tts_text_tokenizer - if cached is None: - model_name_or_path = self.engine_client.model_config.model - tokenizer_path = getattr(self.engine_client.model_config, "tokenizer", None) - if tokenizer_path is None: - tokenizer_path = resolve_glm_tts_tokenizer_path(model_name_or_path) - cached = load_glm_tts_tokenizer( - tokenizer_path, - model_name_or_path=model_name_or_path, - trust_remote_code=bool(getattr(self.engine_client.model_config, "trust_remote_code", False)), - ) - self._glm_tts_text_tokenizer = cached - - frontend = self._glm_tts_text_frontend - if frontend is None: - frontend = GLMTTSTextFrontend() - self._glm_tts_text_frontend = frontend - return cached, frontend - - def _estimate_glm_tts_text_token_len(self, text: str | None, *, add_trailing_space: bool = False) -> int: - """Estimate GLM-TTS normalized text length with the model tokenizer.""" - cached, frontend = self._glm_tts_text_tokenizer_and_frontend() - text = text or "" - normalized = frontend.text_normalize(text) or text - normalized = normalized.strip() - if add_trailing_space and normalized: - normalized = f"{normalized} " - return max(1, len(cached.encode(normalized))) - - def _build_glm_tts_prefill_metadata(self, text: str, prompt_text: str | None) -> dict[str, Any]: - """Build GLM-TTS processor length metadata for additional_information. - - The model preprocess hook runs before postprocess can mirror MM kwargs - into additional_information, so these scalar fields must originate from - the request payload rather than runner-level mm_features. - """ - text_len = self._estimate_glm_tts_text_token_len(text) - prompt_text_len = ( - self._estimate_glm_tts_text_token_len(prompt_text, add_trailing_space=True) if prompt_text else 0 - ) - return { - "glm_tts_text_token_len": [text_len], - "glm_tts_prompt_text_token_len": [prompt_text_len], - "input_len": [prompt_text_len + text_len + 1], - } - - # ---- Ming-flash-omni standalone-talker (TTS) helpers ---- - - def _build_ming_flash_omni_prompt(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: - # request.instructions accepts two forms: - # 1. Plain text: mapped to the caption's 风格 (style) field - # 2. JSON object: parsed and splatted into the caption. Unlocks - # Unknown keys are dropped by `ming_create_instruction`. - caption_fields = self._parse_ming_instruction_fields(request) or {} - - has_spk_emb = request.speaker_embedding is not None - - # TTS path applies ming task type `instruct`. - # voice_name enables talker-side voice preset resolution (e.g. "DB30"). - additional_information: dict[str, Any] = { - "ming_task": "instruct", - "prompt": MING_DEFAULT_PROMPT, - "text": request.input, - "instruction": ming_create_instruction(caption_fields), - "voice_name": request.voice or None, - "use_zero_spk_emb": not has_spk_emb, - "max_decode_steps": request.max_new_tokens or _TTS_MAX_NEW_TOKENS_MAX, - "cfg": 2.0, - "sigma": 0.25, - "temperature": 0.0, - } - if has_spk_emb: - # Passed as plain float list - additional_information["spk_emb"] = list(request.speaker_embedding) - prompt = tokens_input(prompt_token_ids=[0]) - prompt["additional_information"] = additional_information - return prompt - - # ---- Common speech generation helpers ---- - - async def _build_qwen3_tts_request( - self, - request: OpenAICreateSpeechRequest, - ) -> tuple[dict[str, Any], dict[str, Any], str | None]: - """Build prompt + tts_params for Qwen3-TTS. - - Called from ``Qwen3TTSAdapter.build``. Returns - ``(prompt, tts_params, warmup_artifact_key)`` where the warmup key is the - Qwen3-TTS ref-audio artifact tracked after ``generate()``. - """ - qwen3_ref_audio_warmup_artifact_key: str | None = None - tts_params = self._build_tts_params(request) - # Resolve ref_audio (explicit or auto-set for uploaded voices) - # to [[wav_list, sr]] so the model doesn't re-decode base64. - ref_audio_source = request.ref_audio - if ref_audio_source is None and isinstance(tts_params.get("ref_audio"), list): - # Uploaded voice: ref_audio was auto-set as [base64_data_url] - ref_audio_source = tts_params["ref_audio"][0] - if ref_audio_source is not None and isinstance(ref_audio_source, str): - wav_list, sr, cache_key = await self._resolve_ref_audio(ref_audio_source) - tts_params["ref_audio_cache_key"] = cache_key - artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key) - if artifact_key: - tts_params[_QWEN3_TTS_REF_AUDIO_CACHE_KEY] = [artifact_key] - ref_code_length = self._estimate_ref_code_len([wav_list, sr]) - if ref_code_length is not None: - tts_params["ref_code_length"] = [int(ref_code_length)] - if self._qwen3_tts_can_use_ref_audio_artifact_only(tts_params, artifact_key): - logger.debug("Using Qwen3-TTS ref_audio artifact-only path: %s", artifact_key) - else: - tts_params["ref_audio"] = [[wav_list, sr]] - qwen3_ref_audio_warmup_artifact_key = artifact_key - - ph_len = await self._estimate_prompt_len_async(tts_params) - prompt = tokens_input(prompt_token_ids=[1] * ph_len) - prompt["additional_information"] = tts_params - prompt["cache_salt"] = _conditioning_cache_salt(request, tts_params) - return prompt, tts_params, qwen3_ref_audio_warmup_artifact_key - async def _prepare_speech_generation( self, request: OpenAICreateSpeechRequest, @@ -3189,18 +1809,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): stage0_params.extra_args = {} stage0_params.extra_args["tts_local_seed"] = request.seed - if self._tts_model_type == "qwen3_tts" and sampling_params_list: - stage0_params = sampling_params_list[0] - default_seed = getattr(stage0_params, "seed", None) - if default_seed is not None: - import copy - - sampling_params_list = copy.deepcopy(sampling_params_list) - stage0_params = sampling_params_list[0] - if stage0_params.extra_args is None: - stage0_params.extra_args = {} - stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed)) - # When word_timestamps is requested, also ask for the aligner stage's # output so the orchestrator drives the request through the forced-aligner # stage (final_stage_id extends to it). Harmless if no aligner stage exists. diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex.py b/vllm_omni/entrypoints/openai/tts_adapters/audex.py index ab7c7c25e..0e31b633f 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Audex (Nemotron-Labs-Audex-2B) TTS serving adapter.""" +import os from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter @@ -29,6 +31,30 @@ class AudexAdapter(ARTTSAdapter): stage_keys = frozenset({"audex_thinker", "audex_omni"}) name = "audex" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer = None + + def _get_tokenizer(self): + """Load and cache the tokenizer used only by the Audex TTS adapter.""" + if self._tokenizer is None: + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot + + stage = getattr(getattr(self.ctx.server._tts_stage, "engine_args", None), "model_stage", None) + profile, folder = ( + ("full", "checkpoint_folder_full") if stage == "audex_omni" else ("tts", "checkpoint_folder_audiogen") + ) + model_path = os.path.normpath(self.ctx.engine_client.model_config.model) + root = ( + os.path.dirname(model_path) + if os.path.basename(model_path).startswith("checkpoint_folder") + else ensure_audex_snapshot(model_path, profile=profile) + ) + self._tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, folder)) + return self._tokenizer + @classmethod def stage_serves_speech(cls, model_stage: str | None, all_stage_keys: frozenset[str]) -> bool: """``audex_omni`` covers the audex_s2s deployment, whose TTS pass uses @@ -102,7 +128,6 @@ class AudexAdapter(ARTTSAdapter): element of the engine's shared ``default_sampling_params_list`` — see ``_inject_audex_tta_args``). """ - server = self.ctx.server import copy stage0_params = copy.deepcopy(stage0_params) @@ -121,7 +146,7 @@ class AudexAdapter(ARTTSAdapter): cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt if not isinstance(cond_prompt, str) or not cond_prompt: raise ValueError("Audex CFG requires the adapter-built text prompt") - null_prompt = build_null_prompt(cond_prompt, server._get_audex_tokenizer()) + null_prompt = build_null_prompt(cond_prompt, self._get_tokenizer()) extra_args.update( { "cfg_scale": cfg_scale, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py index 33dd2294a..b0af5cdca 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Audex (Nemotron-Labs-Audex-2B) text-to-audio serving adapter. Caption → general audio through the ``audex_tta`` pipeline. @@ -8,6 +9,7 @@ completes the sampling parameters with the RVQ phase-mask contract scale 3.0 — guidance is effectively mandatory for TTA quality). """ +import os from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter @@ -32,6 +34,27 @@ class AudexTTAAdapter(ARTTSAdapter): stage_keys = frozenset({"audex_tta_thinker"}) name = "audex_tta" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer = None + self._tta_rvq = None + + def _get_tokenizer(self): + """Load and cache the tokenizer used only by the Audex TTA adapter.""" + if self._tokenizer is None: + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot + + model_path = os.path.normpath(self.ctx.engine_client.model_config.model) + root = ( + os.path.dirname(model_path) + if os.path.basename(model_path).startswith("checkpoint_folder") + else ensure_audex_snapshot(model_path, profile="tta") + ) + self._tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen")) + return self._tokenizer + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: if not request.input or not request.input.strip(): return "Audex TTA requires a non-empty caption" @@ -95,7 +118,6 @@ class AudexTTAAdapter(ARTTSAdapter): state (``cfg_pair_id``/``cfg_null_prompt``) into it would race concurrent requests and leak stale CFG metadata into later ones. """ - server = self.ctx.server import copy from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt @@ -106,10 +128,10 @@ class AudexTTAAdapter(ARTTSAdapter): raise ValueError("Audex TTA requires the adapter-built caption prompt") stage0_params = copy.deepcopy(stage0_params) - tokenizer = server._get_audex_tokenizer() - if server._audex_tta_rvq is None: + tokenizer = self._get_tokenizer() + if self._tta_rvq is None: phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) - server._audex_tta_rvq = { + self._tta_rvq = { "phase_token_ids": phase_token_ids, "start_tid": start_tid, "end_tid": end_tid, @@ -123,7 +145,7 @@ class AudexTTAAdapter(ARTTSAdapter): if extra_args is None: extra_args = {} stage0_params.extra_args = extra_args - extra_args["tta_rvq"] = server._audex_tta_rvq + extra_args["tta_rvq"] = self._tta_rvq cfg_scale = extra_args.get("cfg_scale") cfg_scale = 3.0 if cfg_scale is None else float(cfg_scale) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index d4316a2fe..ff43b27a5 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -11,8 +11,9 @@ instead of editing the shared serving module in ~10 scattered places. See the RFC for the full design (issue #4327). """ +import hashlib from abc import ABC, abstractmethod -from collections.abc import Callable, Mapping +from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar @@ -37,22 +38,52 @@ DEFAULT_TTS_LANGUAGES = frozenset( } ) -_conditioning_cache_salt_fn: "Callable[..., str] | None" = None +def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict | None = None) -> str: + """Stable hash of the real Stage 0 conditioning for the prefix cache. -def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict) -> str: - """Return the conditioning cache salt for ``request`` + ``tts_params``. + The talker's vLLM prompt is placeholder token ids; the real inputs are + rebuilt from text / ref_audio / ref_text into inputs_embeds. vLLM hashes + token ids (folded with cache_salt) for prefix caching, so without a salt + every request collides and a hit could reuse KV from a semantically + different input. - Lazily imports and caches ``serving_speech._conditioning_cache_salt`` on first - use: the import is deferred to break the adapters<->serving_speech import - cycle, and cached so it resolves once instead of on every ``build()`` call. + Raw request fields alone are not enough for uploaded voices: resolved + conditioning such as ``voice_created_at`` and content-aware ref-audio + cache keys must also be folded in. This distinguishes delete/re-upload of + the same voice and same-path local audio rewrites without hashing decoded + waveform arrays. """ - global _conditioning_cache_salt_fn - if _conditioning_cache_salt_fn is None: - from vllm_omni.entrypoints.openai.serving_speech import _conditioning_cache_salt - - _conditioning_cache_salt_fn = _conditioning_cache_salt - return _conditioning_cache_salt_fn(request, tts_params) + h = hashlib.sha256() + for part in ( + request.input, + request.task_type, + request.language, + request.voice, + request.ref_text, + request.ref_audio, + request.instructions, + request.x_vector_only_mode, + request.speaker_embedding, + ): + h.update(b"\x00") + if part is not None: + h.update(repr(part).encode("utf-8")) + # Fold conditioning derived by adapters and absent from the raw request. + for key in ( + "voice_created_at", + "task_type", + "speaker", + "ref_text", + "x_vector_only_mode", + "ref_audio_cache_key", + "ref_audio_2_cache_key", + ): + h.update(b"\x00") + value = tts_params.get(key) if tts_params is not None else None + if value is not None: + h.update(repr(value).encode("utf-8")) + return h.hexdigest()[:32] def apply_max_new_tokens( diff --git a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py index ec1c145c7..25993ed6e 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py @@ -1,9 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """CosyVoice3 TTS serving adapter.""" import os from typing import TYPE_CHECKING, Any +import numpy as np from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter @@ -14,12 +16,46 @@ if TYPE_CHECKING: logger = init_logger(__name__) +# Without this delimiter the talker re-speaks the reference transcript instead +# of emitting target-only speech (issue #4644). Matches the offline example and +# upstream demo. +_PROMPT_DELIMITER = "<|endofprompt|>" +_PROMPT_PREFIX = f"You are a helpful assistant.{_PROMPT_DELIMITER}" + @register_tts_adapter class CosyVoice3Adapter(ARTTSAdapter): stage_keys = frozenset({"cosyvoice3_talker"}) name = "cosyvoice3" + def __init__(self, ctx) -> None: + super().__init__(ctx) + # Cache the resolved Qwen tokenizer used for dynamic-token sizing. + # Rebuilding it would repeat snapshot resolution and add ~100 ms to the + # TTFP-critical sampling override on every request. + self._tokenizer = None + + async def _build_prompt( + self, request: "OpenAICreateSpeechRequest", *, has_inline_ref_audio: bool = False + ) -> dict[str, Any]: + """Build the multimodal CosyVoice3 voice-cloning prompt.""" + server = self.ctx.server + wav_samples, sr, _ = await server._resolve_ref_audio(request.ref_audio) + ref_text = request.ref_text or "" + if _PROMPT_DELIMITER not in ref_text: + ref_text = f"{_PROMPT_PREFIX}{ref_text}" + mm_kwargs: dict[str, Any] = {"prompt_text": ref_text, "sample_rate": sr} + if request.voice: + voice_lower = request.voice.lower() + if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: + mm_kwargs["voice_name"] = voice_lower + mm_kwargs["voice_created_at"] = server._voice_created_at(voice_lower) + return { + "prompt": request.input, + "multi_modal_data": {"audio": (np.asarray(wav_samples, dtype=np.float32), sr)}, + "mm_processor_kwargs": mm_kwargs, + } + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate CosyVoice3 request parameters. Returns error message or None.""" server = self.ctx.server @@ -51,7 +87,7 @@ class CosyVoice3Adapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = await self.ctx.server._build_cosyvoice3_prompt(request, has_inline_ref_audio=has_inline_ref_audio) + prompt = await self._build_prompt(request, has_inline_ref_audio=has_inline_ref_audio) return PreparedRequest(prompt=prompt, tts_params={}, model_type="cosyvoice3") def apply_sampling_overrides( @@ -76,7 +112,7 @@ class CosyVoice3Adapter(ARTTSAdapter): hf_cfg = server.model_config.hf_config # Build the Qwen tokenizer once per process (resolving the model dir via # snapshot_download at most once) and reuse it across requests. - tokenizer = server._cosyvoice3_tokenizer + tokenizer = self._tokenizer if tokenizer is None: model_path = server.engine_client.model_config.model if not os.path.isdir(model_path): @@ -88,7 +124,7 @@ class CosyVoice3Adapter(ARTTSAdapter): skip_special_tokens=hf_cfg.skip_special_tokens, version=hf_cfg.version, ) - server._cosyvoice3_tokenizer = tokenizer + self._tokenizer = tokenizer _, text_token_len = extract_text_token( request.input, tokenizer, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py index 28e64bc2a..a9840b4a0 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/covo_audio.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """CoVo-Audio serving adapter.""" from typing import TYPE_CHECKING @@ -15,6 +16,10 @@ class CovoAudioAdapter(ARTTSAdapter): stage_keys = frozenset({"fused_thinker_talker"}) name = "covo_audio" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer = None + @classmethod def matches(cls, model_stage: str | None, model_arch: str | None) -> bool: """``fused_thinker_talker`` is a generic stage key that non-CoVo fused @@ -30,5 +35,21 @@ class CovoAudioAdapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = self.ctx.server._build_covo_audio_prompt(request) - return PreparedRequest(prompt=prompt, tts_params={}, model_type="covo_audio") + """Build the tokenized chat prompt expected by Covo-Audio-Chat. + + The model requires a specific system prompt instructing it to + interleave text and audio tokens. Passing token IDs avoids a second + engine-side tokenization step. + """ + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.covo_audio.prompt_utils import build_covo_audio_prompt_token_ids + + if self._tokenizer is None: + model_name = self.ctx.engine_client.model_config.model + try: + self._tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + except Exception as exc: + raise RuntimeError(f"Failed to load Covo-Audio tokenizer from '{model_name}': {exc}") from exc + prompt_ids = build_covo_audio_prompt_token_ids(self._tokenizer, request.input) + return PreparedRequest(prompt={"prompt_token_ids": prompt_ids}, tts_params={}, model_type="covo_audio") diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index a6426a13d..f6dc84d52 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -1,8 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Fish Speech serving adapter (retires the legacy ``_is_fish_speech`` flag).""" +import math from typing import TYPE_CHECKING, Any +import numpy as np +import torch +from vllm.inputs import tokens_input +from vllm.utils.async_utils import make_async + from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ( ARTTSAdapter, @@ -20,6 +27,101 @@ class FishSpeechAdapter(ARTTSAdapter): stage_keys = frozenset({"fish_speech_slow_ar"}) name = "fish_tts" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer = None + self._build_prompt_async = make_async(self._build_prompt, executor=getattr(ctx.server, "_tts_executor", None)) + + @staticmethod + def _estimate_ref_code_len(ref_audio: object) -> int | None: + from vllm_omni.model_executor.models.fish_speech.dac_utils import DAC_HOP_LENGTH, DAC_SAMPLE_RATE + + if not isinstance(ref_audio, (list, tuple)) or len(ref_audio) != 2: + return None + wav, sr = ref_audio + sr = int(sr) + if sr <= 0 or len(wav) <= 0: + return None + return max(1, math.ceil(max(1, math.ceil(len(wav) * DAC_SAMPLE_RATE / sr)) / DAC_HOP_LENGTH)) + + def _get_tokenizer(self): + if self._tokenizer is None: + from transformers import AutoTokenizer + + self._tokenizer = AutoTokenizer.from_pretrained( + self.ctx.engine_client.model_config.model, trust_remote_code=True + ) + return self._tokenizer + + def _estimate_prompt_len(self, text: str, ref_text: str, ref_audio: object) -> int: + from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( + estimate_fish_voice_clone_prompt_len_from_normalized, + ) + + try: + semantic_len = self._estimate_ref_code_len(ref_audio) + if semantic_len is None: + raise ValueError("Failed to estimate Fish Speech semantic token length") + return estimate_fish_voice_clone_prompt_len_from_normalized( + self._get_tokenizer(), text, ref_text, semantic_len + ) + except Exception as exc: + import logging + + logging.getLogger(__name__).warning( + "Failed to estimate Fish Speech prompt length, using fallback 2048: %s", exc + ) + return 2048 + + def _build_prompt( + self, + request: "OpenAICreateSpeechRequest", + ref_audio_data: tuple[list[float], int] | None, + has_inline_ref_audio: bool, + ) -> dict[str, Any]: + """Build either the text-only or structured Fish Speech clone prompt. + + Structured clone metadata uses concrete scalar fields because model-side + preprocess consumes one request directly; text-only metadata retains the + legacy single-item list representation used across EngineCore IPC. + """ + from vllm_omni.model_executor.models.fish_speech.prompt_utils import ( + build_fish_text_only_prompt_ids, + normalize_fish_voice_clone_texts, + ) + + tokenizer = self._get_tokenizer() + if ref_audio_data is None or not request.ref_text: + prompt_ids, normalized_text = build_fish_text_only_prompt_ids(tokenizer, request.input) + info: dict[str, Any] = {"text": [normalized_text]} + if request.max_new_tokens is not None: + info["max_new_tokens"] = [request.max_new_tokens] + prompt = tokens_input(prompt_token_ids=prompt_ids) + prompt["additional_information"] = info + return prompt + + wav_samples, sr = ref_audio_data + normalized_text, normalized_ref_text = normalize_fish_voice_clone_texts(request.input, request.ref_text) + ph_len = self._estimate_prompt_len(normalized_text, normalized_ref_text, ref_audio_data) + info = { + "text": normalized_text, + "ref_text": normalized_ref_text, + "ref_audio_wav": torch.from_numpy(np.asarray(wav_samples, dtype=np.float32)), + "ref_audio_sr": int(sr), + "fish_structured_voice_clone": True, + } + server = self.ctx.server + if request.voice is not None: + voice_lower = request.voice.lower() + if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: + info["voice_name"] = voice_lower + info["voice_created_at"] = server._voice_created_at(voice_lower) + if request.max_new_tokens is not None: + info["max_new_tokens"] = request.max_new_tokens + prompt = tokens_input(prompt_token_ids=[1] * ph_len) + prompt["additional_information"] = info + return prompt + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate Fish Speech request parameters. Returns error message or None.""" server = self.ctx.server @@ -54,9 +156,7 @@ class FishSpeechAdapter(ARTTSAdapter): wav_list, sr, cache_key = await server._resolve_ref_audio(request.ref_audio) ref_audio_data = (wav_list, sr) tts_params["ref_audio_cache_key"] = cache_key - prompt = await server._build_fish_speech_prompt_async( - request, ref_audio_data=ref_audio_data, has_inline_ref_audio=has_inline_ref_audio - ) + prompt = await self._build_prompt_async(request, ref_audio_data, has_inline_ref_audio) prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech") diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index e65edb01b..22dbfc6b6 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """GLM-TTS serving adapter.""" from typing import TYPE_CHECKING, Any +import numpy as np from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter @@ -19,6 +21,72 @@ class GlmTTSAdapter(ARTTSAdapter): stage_keys = frozenset({"glm_tts"}) name = "glm_tts" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._text_tokenizer = None + self._text_frontend = None + + def _text_tokenizer_and_frontend(self): + from vllm_omni.model_executor.models.glm_tts.glm_tts import ( + load_glm_tts_tokenizer, + resolve_glm_tts_tokenizer_path, + ) + from vllm_omni.model_executor.models.glm_tts.text_frontend import GLMTTSTextFrontend + + if self._text_tokenizer is None: + config = self.ctx.engine_client.model_config + tokenizer_path = getattr(config, "tokenizer", None) or resolve_glm_tts_tokenizer_path(config.model) + self._text_tokenizer = load_glm_tts_tokenizer( + tokenizer_path, + model_name_or_path=config.model, + trust_remote_code=bool(getattr(config, "trust_remote_code", False)), + ) + if self._text_frontend is None: + self._text_frontend = GLMTTSTextFrontend() + return self._text_tokenizer, self._text_frontend + + def _estimate_text_token_len(self, text: str | None, *, add_trailing_space: bool = False) -> int: + tokenizer, frontend = self._text_tokenizer_and_frontend() + normalized = (frontend.text_normalize(text or "") or text or "").strip() + if add_trailing_space and normalized: + normalized = f"{normalized} " + return max(1, len(tokenizer.encode(normalized))) + + def _build_prefill_metadata(self, text: str, prompt_text: str | None) -> dict[str, Any]: + text_len = self._estimate_text_token_len(text) + prompt_len = self._estimate_text_token_len(prompt_text, add_trailing_space=True) if prompt_text else 0 + return { + "glm_tts_text_token_len": [text_len], + "glm_tts_prompt_text_token_len": [prompt_len], + "input_len": [prompt_len + text_len + 1], + } + + async def _build_prompt( + self, request: "OpenAICreateSpeechRequest", *, has_inline_ref_audio: bool = False + ) -> dict[str, Any]: + """Build GLM-TTS multimodal voice-cloning input and prefill metadata. + + AR preprocess constructs PromptText, Text, BOA and prompt speech tokens; + the DiT consumes the resulting prompt tokens, features and embedding as + conditioning. + """ + server = self.ctx.server + if request.ref_audio is None or not request.ref_text: + raise ValueError("GLM-TTS requires ref_audio and ref_text for voice cloning.") + wav_samples, sr, _ = await server._resolve_ref_audio(request.ref_audio) + mm_kwargs: dict[str, Any] = {"prompt_text": request.ref_text} + if request.voice: + voice_lower = request.voice.lower() + if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: + mm_kwargs["voice_name"] = voice_lower + mm_kwargs["voice_created_at"] = server._voice_created_at(voice_lower) + return { + "prompt": request.input, + "multi_modal_data": {"audio": (np.asarray(wav_samples, dtype=np.float32), int(sr))}, + "mm_processor_kwargs": mm_kwargs, + "additional_information": self._build_prefill_metadata(request.input, request.ref_text), + } + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate GLM-TTS request — requires ref_audio for voice cloning.""" server = self.ctx.server @@ -46,7 +114,7 @@ class GlmTTSAdapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = await self.ctx.server._build_glm_tts_prompt(request, has_inline_ref_audio=has_inline_ref_audio) + prompt = await self._build_prompt(request, has_inline_ref_audio=has_inline_ref_audio) return PreparedRequest(prompt=prompt, tts_params={}, model_type="glm_tts") def _load_supported_speakers(self) -> set[str]: @@ -71,9 +139,7 @@ class GlmTTSAdapter(ARTTSAdapter): if isinstance(text_len_value, list) and text_len_value: text_len_value = text_len_value[0] text_token_len = ( - int(text_len_value) - if text_len_value is not None - else server._estimate_glm_tts_text_token_len(request.input) + int(text_len_value) if text_len_value is not None else self._estimate_text_token_len(request.input) ) hf_cfg = server.model_config.hf_config min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py index 74ec591cc..207d8b47b 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py @@ -1,8 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Higgs-Audio v2 serving adapter.""" +import asyncio from typing import TYPE_CHECKING, Any +import numpy as np +from vllm.inputs import tokens_input + from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens @@ -12,6 +17,89 @@ if TYPE_CHECKING: @register_tts_adapter class HiggsAudioV2Adapter(ARTTSAdapter): + @property + def engine_client(self): + return self.ctx.engine_client + + async def _resolve_ref_audio(self, ref_audio: str): + return await self.ctx.server._resolve_ref_audio(ref_audio) + + async def _build_higgs_audio_v2_params(self, request: "OpenAICreateSpeechRequest"): + """Build prompt_token_ids for higgs_audio_v2 via the upstream processor. + + Plain-text path: runs ``build_plain_text_prompt`` and returns the + token-only prompt. Voice-clone path (``ref_audio`` + ``ref_text``): + resolves the reference clip via ``_resolve_ref_audio``, runs + ``build_voice_clone_prompt`` (which encodes the clip through HF's + ``HiggsAudioV2TokenizerModel`` loaded from the k2-fsa/OmniVoice + ``audio_tokenizer/`` subdirectory), and attaches the encoded + ``audio_input_ids`` + ``audio_input_ids_mask`` tensors via + ``additional_information`` so the talker substitutes them at the + prompt-side audio placeholders. + """ + from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import ( + build_plain_text_prompt, + build_voice_clone_prompt, + input_ids_to_python_list, + ) + + processor = await self._resolve_higgs_audio_v2_processor() + + if request.ref_audio is None: + inputs = await asyncio.to_thread(build_plain_text_prompt, processor, request.input) + prompt_token_ids = input_ids_to_python_list(inputs) + return tokens_input(prompt_token_ids=prompt_token_ids) + + wav_list, sr, _ = await self._resolve_ref_audio(request.ref_audio) + wav = np.asarray(wav_list, dtype=np.float32) + out = await asyncio.to_thread( + build_voice_clone_prompt, + processor, + request.input, + wav, + int(sr), + request.ref_text or "", + ) + prompt = tokens_input(prompt_token_ids=out["prompt_token_ids"]) + # Pass tensors at the top level of additional_information (NOT list- + # wrapped). ``vllm_omni.data_entry_keys.serialize_payload`` routes + # bare ``torch.Tensor`` values through ``_serialize_tensor``; a list + # containing tensors would fall into the ``list_data`` field which + # msgspec cannot serialize and the tensors would be dropped over the + # process boundary (silent voice-clone failure). + prompt["additional_information"] = { + "audio_input_ids": out["audio_input_ids"], + "audio_input_ids_mask": out["audio_input_ids_mask"], + } + return prompt + + async def _resolve_higgs_audio_v2_processor(self): + """Lazy-load the AutoProcessor for higgs_audio_v2 (once per serving instance).""" + cached = getattr(self, "_higgs_audio_v2_processor", None) + if cached is not None: + return cached + + from transformers import AutoProcessor + + model_path = None + for stage in self.engine_client.stage_configs: + model_path = getattr(getattr(stage, "engine_args", None), "model", None) + if model_path: + break + if model_path is None: + # Fallback: the orchestrator stores the served model id on the engine + # itself (set by AsyncOmniEngine.__init__). Stage-level engine_args + # may not surface ``model`` when the deploy yaml doesn't set it per + # stage (the CLI-passed model id is the single source of truth). + model_path = getattr(self.engine_client, "model", None) + if model_path is None: + raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs") + processor = AutoProcessor.from_pretrained(model_path) + self._higgs_audio_v2_processor = processor + return processor + + # ---- higgs-audio v3 ---- + stage_keys = frozenset({"higgs_audio_v2"}) name = "higgs_audio_v2" @@ -91,7 +179,7 @@ class HiggsAudioV2Adapter(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: server = self.ctx.server - prompt = await server._build_higgs_audio_v2_params(request) + prompt = await self._build_higgs_audio_v2_params(request) if request.voice: voice_lower = request.voice.lower() if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py index 411d97f31..eb00fe784 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py @@ -1,10 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Higgs-Audio v3 serving adapter.""" +import asyncio +from collections import OrderedDict from typing import TYPE_CHECKING, Any +import numpy as np +import torch +from vllm.inputs import tokens_input + from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens +from vllm_omni.entrypoints.openai.tts_adapters.base import ( + ARTTSAdapter, + PreparedRequest, + apply_max_new_tokens, + conditioning_cache_salt, +) + +_REF_CODE_CACHE_MAX_ENTRIES = 256 +_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024 if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -12,6 +27,159 @@ if TYPE_CHECKING: @register_tts_adapter class HiggsAudioV3Adapter(ARTTSAdapter): + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._higgs_audio_v3_ref_code_cache: OrderedDict[str, tuple[torch.Tensor, int]] = OrderedDict() + self._higgs_audio_v3_ref_code_cache_bytes = 0 + self._higgs_audio_v3_ref_code_inflight: dict[str, asyncio.Task[torch.Tensor]] = {} + + @property + def engine_client(self): + return self.ctx.engine_client + + async def _resolve_ref_audio(self, ref_audio: str): + return await self.ctx.server._resolve_ref_audio(ref_audio) + + def _get_resolved_ref_audio_artifact_key(self, cache_key: str): + return self.ctx.server._get_resolved_ref_audio_artifact_key(cache_key) + + async def _build_higgs_audio_v3_params(self, request: "OpenAICreateSpeechRequest"): + """Build prompt_token_ids for higgs_audio_v3. + + Plain-text path: builds ``[tts, text, tokens, audio]``. + Voice-clone path: encodes reference audio, applies delay pattern, and + records reference-audio prompt positions while submitting valid token IDs. + """ + adapter = await self._resolve_higgs_audio_v3_adapter() + + if request.ref_audio is None: + prompt_ids = adapter.build_prompt(request.input) + return tokens_input(prompt_token_ids=prompt_ids) + + # Voice clone + from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import ( + apply_delay_pattern, + encode_reference_audio, + ) + + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) + artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key) + wav = np.asarray(wav_list, dtype=np.float32) + ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes( + artifact_key, + wav, + int(sr), + encode_reference_audio, + apply_delay_pattern, + ) + del cache_hit, inflight_wait + + prompt_ids = adapter.build_prompt( + request.input, + num_ref_tokens=int(ref_codes_delayed.shape[0]), + reference_text=request.ref_text or None, + ) + prompt_ids, audio_placeholder_positions = adapter.prepare_prompt_for_engine(prompt_ids) + prompt = tokens_input(prompt_token_ids=prompt_ids) + import torch + + prompt["additional_information"] = { + "audio_input_ids": ref_codes_delayed.to(torch.long), + "audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool), + "audio_placeholder_positions": audio_placeholder_positions, + "ref_audio_cache_key": cache_key, + } + prompt["cache_salt"] = conditioning_cache_salt(request, prompt["additional_information"]) + return prompt + + async def _resolve_higgs_audio_v3_ref_codes( + self, + artifact_key: str | None, + wav: np.ndarray, + sr: int, + encode_reference_audio, + apply_delay_pattern, + ) -> tuple[torch.Tensor, bool, bool]: + ref_codes_delayed = self._get_higgs_audio_v3_ref_codes(artifact_key) + if ref_codes_delayed is not None: + return ref_codes_delayed, True, False + if not artifact_key: + ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr) + return apply_delay_pattern(ref_codes_raw), False, False + + task = self._higgs_audio_v3_ref_code_inflight.get(artifact_key) + if task is not None: + return (await task).clone(), False, True + + async def _encode_and_cache() -> torch.Tensor: + ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr) + delayed = apply_delay_pattern(ref_codes_raw) + self._put_higgs_audio_v3_ref_codes(artifact_key, delayed) + cached = self._get_higgs_audio_v3_ref_codes(artifact_key) + return cached if cached is not None else delayed.detach().to("cpu", dtype=torch.long).contiguous() + + task = asyncio.create_task(_encode_and_cache()) + self._higgs_audio_v3_ref_code_inflight[artifact_key] = task + try: + return (await task).clone(), False, False + finally: + if self._higgs_audio_v3_ref_code_inflight.get(artifact_key) is task: + self._higgs_audio_v3_ref_code_inflight.pop(artifact_key, None) + + def _get_higgs_audio_v3_ref_codes(self, artifact_key: str | None) -> torch.Tensor | None: + if not artifact_key: + return None + cached = self._higgs_audio_v3_ref_code_cache.get(artifact_key) + if cached is None: + return None + self._higgs_audio_v3_ref_code_cache.move_to_end(artifact_key) + return cached[0].clone() + + def _put_higgs_audio_v3_ref_codes(self, artifact_key: str, codes: torch.Tensor) -> None: + if _REF_CODE_CACHE_MAX_ENTRIES <= 0 or _REF_CODE_CACHE_MAX_BYTES <= 0 or not artifact_key: + return + cached_codes = codes.detach().to("cpu", dtype=torch.long).contiguous() + size = int(cached_codes.numel() * cached_codes.element_size()) + if size > _REF_CODE_CACHE_MAX_BYTES: + return + previous = self._higgs_audio_v3_ref_code_cache.pop(artifact_key, None) + if previous is not None: + self._higgs_audio_v3_ref_code_cache_bytes -= previous[1] + self._higgs_audio_v3_ref_code_cache[artifact_key] = (cached_codes, size) + self._higgs_audio_v3_ref_code_cache_bytes += size + while len(self._higgs_audio_v3_ref_code_cache) > _REF_CODE_CACHE_MAX_ENTRIES: + _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False) + self._higgs_audio_v3_ref_code_cache_bytes -= old_size + while self._higgs_audio_v3_ref_code_cache_bytes > _REF_CODE_CACHE_MAX_BYTES: + _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False) + self._higgs_audio_v3_ref_code_cache_bytes -= old_size + + async def _resolve_higgs_audio_v3_adapter(self): + """Lazy-load the tokenizer adapter for higgs_audio_v3.""" + cached = getattr(self, "_higgs_audio_v3_adapter", None) + if cached is not None: + return cached + + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import ( + HiggsAudioV3TokenizerAdapter, + ) + + model_path = None + for stage in self.engine_client.stage_configs: + model_path = getattr(getattr(stage, "engine_args", None), "model", None) + if model_path: + break + if model_path is None: + model_path = getattr(self.engine_client, "model", None) + if model_path is None: + raise RuntimeError("higgs_audio_v3 serving could not resolve model path") + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + adapter = HiggsAudioV3TokenizerAdapter(tokenizer) + self._higgs_audio_v3_adapter = adapter + return adapter + stage_keys = frozenset({"higgs_audio_v3"}) name = "higgs_audio_v3" @@ -34,7 +202,7 @@ class HiggsAudioV3Adapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = await self.ctx.server._build_higgs_audio_v3_params(request) + prompt = await self._build_higgs_audio_v3_params(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v3") def apply_sampling_overrides( diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py index bc084d6b4..05cd3a64e 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py @@ -1,12 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """MingFlashOmniTTS serving adapter.""" from typing import TYPE_CHECKING +from vllm.inputs import tokens_input from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import parse_ming_instruction_fields if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -61,8 +64,36 @@ class MingFlashOmniTTSAdapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: + """Build the standalone Ming Flash TTS prompt.""" + from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import ( + DEFAULT_PROMPT, + create_instruction, + ) + server = self.ctx.server - prompt = server._build_ming_flash_omni_prompt(request) + # ``instructions`` accepts either plain style text or a JSON object. + # Unknown caption keys are filtered by ``create_instruction``. + caption_fields = parse_ming_instruction_fields(server, request) or {} + has_spk_emb = request.speaker_embedding is not None + # Standalone TTS uses the Ming ``instruct`` task. ``voice_name`` lets + # the talker resolve registered presets such as DB30. + info = { + "ming_task": "instruct", + "prompt": DEFAULT_PROMPT, + "text": request.input, + "instruction": create_instruction(caption_fields), + "voice_name": request.voice or None, + "use_zero_spk_emb": not has_spk_emb, + "max_decode_steps": request.max_new_tokens or self.max_new_tokens_max, + "cfg": 2.0, + "sigma": 0.25, + "temperature": 0.0, + } + if has_spk_emb: + # Keep embeddings as plain floats for EngineCore serialization. + info["spk_emb"] = list(request.speaker_embedding) + prompt = tokens_input(prompt_token_ids=[0]) + prompt["additional_information"] = info return PreparedRequest(prompt=prompt, tts_params={}, model_type="ming_flash_omni_tts") def _load_supported_speakers(self) -> set[str]: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index 613968b8c..8dc5731a0 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Ming-TTS (dense) serving adapter.""" +import json from typing import TYPE_CHECKING, Any +import torch +from vllm.inputs import tokens_input from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM if TYPE_CHECKING: @@ -15,8 +20,140 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def parse_ming_instruction_fields(server, request, *, include_voice=False, plain_text_passthrough=False): + instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None + instruction_dict: dict[str, Any] = {} + voice_lower = request.voice.lower() if isinstance(request.voice, str) else None + if include_voice and request.voice and not (voice_lower and voice_lower in server.uploaded_speakers): + instruction_dict["IP"] = request.voice + if instruction_text: + try: + parsed = json.loads(instruction_text) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + instruction_dict.update(parsed) + elif instruction_dict or not plain_text_passthrough: + instruction_dict["风格"] = instruction_text + else: + return instruction_text + return instruction_dict or None + + @register_tts_adapter class MingTTSAdapter(ARTTSAdapter): + def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate): + from torchaudio.functional import resample as resample_audio + + from vllm_omni.model_executor.models.ming_tts.config_ming_tts import SAMPLE_RATE + + waveform = torch.as_tensor(wav_samples, dtype=torch.float32).reshape(1, -1) + if int(sample_rate) != SAMPLE_RATE: + waveform = resample_audio(waveform, int(sample_rate), SAMPLE_RATE) + return waveform + + def _build_ming_reference_waveforms( + self, + ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None, + ) -> tuple[list[torch.Tensor], list[int]]: + if ref_audio_data is None: + return [], [] + items = ref_audio_data if isinstance(ref_audio_data, list) else [ref_audio_data] + waveforms = [torch.as_tensor(samples, dtype=torch.float32).reshape(1, -1) for samples, _ in items] + sample_rates = [int(sample_rate) for _, sample_rate in items] + if any(sample_rate <= 0 for sample_rate in sample_rates): + raise ValueError(f"Ming reference audio sample rates must be positive, got {sample_rates}") + return waveforms, sample_rates + + def _parse_ming_instruction(self, request: "OpenAICreateSpeechRequest") -> Any: + """Build a Ming instruction payload from OpenAI speech fields.""" + return parse_ming_instruction_fields( + self.ctx.server, + request, + include_voice=True, + plain_text_passthrough=True, + ) + + def _build_ming_dense_prompt( + self, + request: "OpenAICreateSpeechRequest", + *, + ref_audio_data: tuple[list[float], int] | list[tuple[list[float], int]] | None = None, + voice_name: str | None = None, + voice_created_at: int = 0, + ) -> dict[str, Any]: + """Build a Ming dense prompt directly from the OpenAI speech request.""" + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( + KEY_MAX_DECODE_STEPS, + KEY_SPEAKER_SAMPLE_RATES, + KEY_SPEAKER_WAVEFORM, + KEY_SPEAKER_WAVEFORM_LENGTHS, + ) + from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt + + if self.ctx.server._tts_tokenizer is None: + model_name = self.ctx.engine_client.model_config.model + trust_remote_code = bool(getattr(self.ctx.engine_client.model_config, "trust_remote_code", False)) + self.ctx.server._tts_tokenizer = AutoTokenizer.from_pretrained( + model_name, trust_remote_code=trust_remote_code + ) + + ref_text = request.ref_text + reference_waveforms, reference_sample_rates = self._build_ming_reference_waveforms(ref_audio_data) + reference_waveform = torch.cat(reference_waveforms, dim=-1) if reference_waveforms else None + prompt_waveforms = ( + [ + self._coerce_ming_prompt_waveform(waveform, sample_rate) + for waveform, sample_rate in zip(reference_waveforms, reference_sample_rates, strict=True) + ] + if ref_text is not None + else [] + ) + prompt_waveform = torch.cat(prompt_waveforms, dim=-1) if prompt_waveforms else None + speaker_embedding = request.speaker_embedding + pending_speaker_extraction = speaker_embedding is None and reference_waveform is not None + use_zero_spk_emb = not pending_speaker_extraction and prompt_waveform is None and speaker_embedding is None + + runtime_controls = {} + if request.max_new_tokens is not None: + runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens + + prompt_dict = build_ming_dense_prompt( + self.ctx.server._tts_tokenizer, + # bgm / music-prompt mode not supported online; + # requires prompt_mode API extension (deferred). + prompt=MING_DEFAULT_PROMPT, + text=request.input, + runtime_controls=runtime_controls or None, + instruction=self._parse_ming_instruction(request), + prompt_text=ref_text, + prompt_waveform=prompt_waveform, + speaker_embedding=speaker_embedding, + speaker_count=len(reference_waveforms) if pending_speaker_extraction else None, + use_zero_spk_emb=use_zero_spk_emb, + ) + additional_information = prompt_dict["additional_information"] + if pending_speaker_extraction: + additional_information[KEY_SPEAKER_WAVEFORM] = reference_waveform + additional_information[KEY_SPEAKER_WAVEFORM_LENGTHS] = torch.tensor( + [int(waveform.shape[-1]) for waveform in reference_waveforms], + dtype=torch.int32, + ) + additional_information[KEY_SPEAKER_SAMPLE_RATES] = torch.tensor( + reference_sample_rates, + dtype=torch.int32, + ) + if voice_name: + additional_information["voice_name"] = voice_name + additional_information["voice_created_at"] = int(voice_created_at) + prompt = tokens_input(prompt_token_ids=prompt_dict["prompt_token_ids"]) + prompt["prompt"] = prompt_dict["prompt"] + prompt["text"] = prompt_dict["text"] + prompt["additional_information"] = prompt_dict["additional_information"] + return prompt + # Ming dense has no dedicated model_stage value, so both stage discovery and # model-type detection go through the architecture. ``ming_flash_omni_tts`` # is the model that owns the ``ming_tts`` *stage key*, and is matched first. @@ -66,7 +203,7 @@ class MingTTSAdapter(ARTTSAdapter): elif ref_audio_source is not None and isinstance(ref_audio_source, str): wav_list, sr, _ = await server._resolve_ref_audio(ref_audio_source) ref_audio_data = (wav_list, sr) - prompt = server._build_ming_dense_prompt( + prompt = self._build_ming_dense_prompt( request, ref_audio_data=ref_audio_data, voice_name=uploaded_audio_voice, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 3d6e99552..e264c7af5 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -6,7 +6,7 @@ Both variants share the same build/validate flow (``_build_moss_tts_params`` handles each); they are registered under distinct model-type names. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm.inputs import tokens_input @@ -23,6 +23,255 @@ if TYPE_CHECKING: class _MossTTSAdapterBase(ARTTSAdapter): + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._moss_variant = None if self.name == "moss_tts_nano" else self._detect_moss_variant() + self._moss_processor_cache = None + + @property + def engine_client(self): + return self.ctx.engine_client + + @property + def uploaded_speakers(self): + return self.ctx.server.uploaded_speakers + + @property + def _speaker_cache(self): + return self.ctx.server._speaker_cache + + async def _resolve_ref_audio(self, ref_audio: str): + return await self.ctx.server._resolve_ref_audio(ref_audio) + + def _voice_created_at(self, voice: str) -> int: + return self.ctx.server._voice_created_at(voice) + + def _get_resolved_ref_audio_artifact_key(self, cache_key: str): + return self.ctx.server._get_resolved_ref_audio_artifact_key(cache_key) + + def _get_moss_ref_encoder(self): + """Lazily build the per-server MOSS reference-audio encoder once.""" + cached = getattr(self, "_moss_ref_encoder", None) + if cached is not None: + return cached + from vllm_omni.model_executor.models.moss_tts.reference_encoder import build_reference_encoder + + # The variant's encode geometry (n_vq, working sample rate) is derived + # inside the model package; this layer only supplies the processor and + # the process-wide speaker cache. + encoder = build_reference_encoder( + self._get_moss_processor(), + variant=self._moss_variant, + speaker_cache=self._speaker_cache, + ) + self._moss_ref_encoder = encoder + return encoder + + async def _encode_moss_references( + self, + request: "OpenAICreateSpeechRequest", + *, + has_inline_ref_audio: bool, + two_speaker: bool, + ) -> tuple[list, dict[int, str]]: + """Encode the request's reference clip(s) into MOSS RVQ code tensors. + + Reference encoding + speaker caching lives in the model package + (moss_tts.reference_encoder), mirroring Fish Speech / CosyVoice3 / + Qwen3-TTS which keep reference handling with the model rather than in + this shared serving file. This method only resolves the serving-side + inputs the model helper needs: the generic audio resolver/artifact-key + lookups and whether the request may use the named-voice cache. + + Returns ``(codes_per_speaker, resolve_keys)`` where ``resolve_keys`` + maps the reference slot (0 = ref_audio, 1 = ref_audio_2) to its + content-aware resolve key, for salting the KV prefix cache. + """ + from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_request_references + + # Named-voice caching is only valid for uploaded speakers without an + # inline ref_audio: ``request.voice`` plus a file/URL would otherwise + # key on (name, created_at=0) and skip the content-aware resolve. + raw_voice = getattr(request, "voice", None) + raw_voice = raw_voice.strip() if isinstance(raw_voice, str) else "" + voice_lower = raw_voice.lower() + use_named_voice = bool(voice_lower) and voice_lower in self.uploaded_speakers and not has_inline_ref_audio + voice = voice_lower if use_named_voice else "" + voice_created = self._voice_created_at(voice) if voice else 0 + + return await encode_request_references( + self._get_moss_ref_encoder(), + request.ref_audio, + request.ref_audio_2 if two_speaker else None, + resolve_ref_audio=self._resolve_ref_audio, + get_artifact_key=self._get_resolved_ref_audio_artifact_key, + voice_name=voice or None, + voice_created_at=voice_created, + ) + + def _detect_moss_variant(self) -> str: + """Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS + variant family (tts, ttsd, sound_effect, voice_generator, realtime). + + Detection key is the HF repo path / model_name; matches + ``_try_resolve_omni_model_type`` in entrypoints/utils.py so users get + consistent behaviour no matter how they launched the server (--model + OpenMOSS-Team/MOSS-TTSD-v1.0 vs --deploy-config moss_ttsd.yaml). + """ + try: + name = (self.engine_client.model_config.model or "").lower().replace("-", "").replace("_", "") + except Exception: + name = "" + if "realtime" in name: + return "realtime" + if "local" in name: + return "local" + if "ttsd" in name: + return "ttsd" + if "soundeffect" in name: + return "sound_effect" + if "voicegenerator" in name: + return "voice_generator" + return "tts" + + def _get_moss_processor(self): + """Lazily load the upstream MOSS-TTS processor once per server. + + Cached on ``self._moss_processor_cache``. The processor owns its own + audio_tokenizer (~1.6 B params); we keep it on CPU so it doesn't + compete with the talker (~8 GiB) and codec (~7 GiB) for our 96 GiB + GPU — per-request ref-audio encoding is fast enough on CPU. + """ + cached = getattr(self, "_moss_processor_cache", None) + if cached is not None: + return cached + from transformers import AutoProcessor + + model_id = self.engine_client.model_config.model + proc = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) + if hasattr(proc, "audio_tokenizer"): + proc.audio_tokenizer = proc.audio_tokenizer.to("cpu").eval() + self._moss_processor_cache = proc + return proc + + async def _build_moss_tts_params( + self, + request: "OpenAICreateSpeechRequest", + *, + has_inline_ref_audio: bool = False, + ) -> dict[str, Any]: + """Build the talker prompt + ``additional_information`` payload for any + MOSS-TTS-family request (nano + 5 full variants). + + For the legacy ``moss_tts_nano`` model_type, keeps the original nano + contract (``{text, mode=voice_clone, prompt_audio_array}``); the + caller still uses a ``[1]`` placeholder prompt. + + For the full MOSS-TTS family (``MossTTSDelay*`` / ``MossTTSRealtime*``) + we **call the upstream processor server-side** to produce the unified + ``(text_ids, audio_codes)`` shape the talker actually consumes — same + flow as ``examples/.../moss_tts/end2end.py:_build_unified_codes``. + Returns ``{prompt_token_ids: list[int], codes.ref: torch.LongTensor, + max_new_frames, ...}``. The caller treats ``prompt_token_ids`` as the + prompt and forwards the rest as ``additional_information``. + """ + import torch # local to avoid pulling torch at module import time + + v = self._moss_variant + + # ---- Legacy nano path (unchanged) ---- + if v is None: # moss_tts_nano + params: dict[str, Any] = { + "text": [request.input or ""], + "mode": ["voice_clone"], + } + if request.max_new_tokens is not None: + params["max_new_frames"] = [request.max_new_tokens] + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) + params["prompt_audio_array"] = [[wav_list, sr]] + params["ref_audio_cache_key"] = cache_key + return params + + # ---- MOSS-TTS-Realtime: keep the old prompt_audio_array path ---- + # ``AutoProcessor.from_pretrained`` doesn't auto-discover + # ``MossTTSRealtimeProcessor`` (no ``processor_config.json`` in the + # snapshot), and Realtime's prompt format diverges from MossTTSDelay + # (16-channel grid, separate per-step text feed). The + # ``prompt_audio_array`` shape lines up well enough with what the + # talker reads for short prompts; full Realtime support needs a + # separate processor.from_module path which we don't wire here. + if v == "realtime": + params: dict[str, Any] = { + "text": [request.input or ""], + "mode": ["voice_clone"], + } + if request.max_new_tokens is not None: + params["max_new_frames"] = [request.max_new_tokens] + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) + params["prompt_audio_array"] = [[wav_list, sr]] + params["ref_audio_cache_key"] = cache_key + return params + + # ---- MossTTSDelay family (tts/ttsd/sound_effect/voice_generator) + # and MOSS-TTS-Local-Transformer-v1.5: call the upstream processor + # server-side to produce unified codes. Local-v1.5 ships its own + # AutoProcessor (processor_config.json + processing_moss_tts.py) and + # reuses this exact build_user_message/encode_audios_from_wav path in + # the offline example (examples/.../moss_tts/end2end.py: + # _build_unified_codes) -- it is NOT in the same boat as Realtime + # (no processor_config.json there), so it must not fall back to the + # prompt_audio_array path above (which the talker's preprocess() + # never reads -- info_dict["codes"]["ref"] is the only thing it + # consumes, so skipping this path silently drops all voice-clone + # conditioning and produces unconditioned/garbage audio online). ---- + proc = self._get_moss_processor() + + user_kwargs: dict[str, Any] = {"text": request.input or ""} + resolve_keys: dict[int, str] = {} + if v in ("tts", "local", "ttsd"): + user_kwargs["reference"], resolve_keys = await self._encode_moss_references( + request, + has_inline_ref_audio=has_inline_ref_audio, + two_speaker=(v == "ttsd"), + ) + elif v == "sound_effect": + user_kwargs["text"] = request.input or "" # may be empty + user_kwargs["ambient_sound"] = request.ambient_sound or "" + if request.duration_seconds is not None: + user_kwargs["tokens"] = max(1, int(float(request.duration_seconds) * 12.5)) + elif request.max_new_tokens is not None: + user_kwargs["tokens"] = int(request.max_new_tokens) + elif v == "voice_generator": + user_kwargs["instruction"] = request.instructions or "" + + # Optional language tag for the spoken-text variants. MOSS-TTS-v1.5's + # headline improvement is multilingual synthesis when the language is + # given (build_user_message(..., language=...)); 1.0 ignores it + # gracefully. Sound-effect output is non-verbal, so skip it there. + if v in ("tts", "local", "ttsd", "voice_generator") and getattr(request, "language", None): + user_kwargs["language"] = request.language + + # Build the unified-codes prompt: (L, 1+n_vq) where col 0 is text/special + # tokens and cols 1..n_vq are the delay-pattern audio code grid (mostly + # audio_pad_code outside the reference block). + user_msg = proc.build_user_message(**user_kwargs) + batch = proc(conversations=[[user_msg]], mode="generation") + unified = batch["input_ids"][0] # torch.LongTensor (L, 1+n_vq) + text_ids: list[int] = unified[:, 0].tolist() + audio_codes: torch.Tensor = unified[:, 1:].contiguous().to(torch.int64) + + params: dict[str, Any] = { + "prompt_token_ids": text_ids, + "codes": {"ref": audio_codes}, + } + if request.max_new_tokens is not None: + params["max_new_frames"] = [request.max_new_tokens] + if 0 in resolve_keys: + params["ref_audio_cache_key"] = resolve_keys[0] + if 1 in resolve_keys: + params["ref_audio_2_cache_key"] = resolve_keys[1] + return params + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate any MOSS-TTS-family request (nano + 5 full variants). @@ -42,10 +291,10 @@ class _MossTTSAdapterBase(ARTTSAdapter): if not request.input or not request.input.strip(): # SoundEffect can legitimately have empty input (just ambient_sound). - if server._moss_variant != "sound_effect": + if self._moss_variant != "sound_effect": return "Input text cannot be empty" - v = server._moss_variant + v = self._moss_variant if v in (None, "tts", "realtime", "local"): if request.ref_audio is None: label = ( @@ -92,7 +341,7 @@ class _MossTTSAdapterBase(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: server = self.ctx.server - tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio) + tts_params = await self._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio) if request.voice: voice_lower = request.voice.lower() if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index 48d019eeb..3a2bb9860 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -2,12 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Qwen3-TTS serving adapter.""" +import math from collections.abc import Mapping from pathlib import Path from typing import TYPE_CHECKING, Any import regex as re +from vllm.inputs import tokens_input from vllm.logger import init_logger +from vllm.utils.async_utils import make_async from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ( @@ -15,6 +18,7 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ( ARTTSAdapter, PreparedRequest, TTSGenerationError, + conditioning_cache_salt, ) from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_precomputed_speakers from vllm_omni.utils.speaker_cache import validate_qwen3_tts_profile @@ -23,6 +27,7 @@ if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest logger = init_logger(__name__) +_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key" QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY = "_qwen3_tts_effective_max_tokens" _MIN_CODEC_FRAMES = 192 @@ -45,6 +50,33 @@ class Qwen3TTSAdapter(ARTTSAdapter): name = "qwen3_tts" supported_output_sample_rates = frozenset({8000, 24000}) + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._estimate_prompt_len_async = make_async( + self._estimate_prompt_len, executor=getattr(ctx.server, "_tts_executor", None) + ) + + def _estimate_ref_code_len(self, ref_audio: object) -> int | None: + codec_frame_rate = self.capabilities.codec_frame_rate + if codec_frame_rate is None: + return None + try: + item = ref_audio + while isinstance(item, list) and item: + if len(item) == 2 and isinstance(item[1], (int, float)): + break + item = item[0] + if not isinstance(item, (list, tuple)) or len(item) != 2: + return None + wav, sr = item + sr = int(sr) + n_samples = len(wav) if hasattr(wav, "__len__") else wav.shape[-1] + if sr <= 0 or n_samples <= 0: + return None + return math.ceil(n_samples / sr * codec_frame_rate) + except Exception: + return None + def _get_model_variant(self) -> str | None: """Return the task supported by the loaded Qwen3-TTS checkpoint. @@ -223,15 +255,208 @@ class Qwen3TTSAdapter(ARTTSAdapter): return None + def _build_tts_params(self, request: "OpenAICreateSpeechRequest") -> dict[str, Any]: + """Build TTS parameters from request. + + Processes each parameter if present, skips if not. + Values are wrapped in lists as required by the model. + """ + server = self.ctx.server + + params: dict[str, Any] = {} + + # Text content (always required) + params["text"] = [request.input] + + # Task type + if request.task_type is not None: + params["task_type"] = [request.task_type] + else: + params["task_type"] = ["CustomVoice"] + + # Language + if request.language is not None: + params["language"] = [request.language] + else: + params["language"] = ["Auto"] + + # Speaker (voice) + if request.voice is not None: + voice_lower = request.voice.lower() + precomputed_speakers = self.capabilities.precomputed_speakers + params["speaker"] = [request.voice] + params["voice_created_at"] = [server._voice_created_at(voice_lower)] + + # Uploaded voices use task_type="Base" (CustomVoice requires built-in spk_id). + # If ref_text was provided at upload time, use in-context cloning; otherwise x_vector only. + if voice_lower in server.uploaded_speakers and request.ref_audio is None: + speaker_info = server.uploaded_speakers[voice_lower] + + # Check if this voice was uploaded with a pre-computed embedding. + # Populate request.speaker_embedding so the existing code path + # (below) handles voice_clone_prompt and x_vector_only_mode. + embedding = server._get_uploaded_speaker_embedding(request.voice) + if embedding is not None: + request.speaker_embedding = embedding + params["speaker"] = [voice_lower] + params["task_type"] = ["Base"] + logger.info("Auto-set speaker_embedding for uploaded voice: %s", request.voice) + else: + audio_data = server._get_uploaded_audio_data(request.voice) + if not audio_data: + raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted") + stored_ref_text = speaker_info.get("ref_text") + params["speaker"] = [voice_lower] + params["ref_audio"] = [audio_data] + params["task_type"] = ["Base"] + if stored_ref_text: + params["ref_text"] = [stored_ref_text] + params["x_vector_only_mode"] = [False] + else: + params["x_vector_only_mode"] = [True] + logger.info( + "Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text) + ) + elif voice_lower in precomputed_speakers and request.ref_audio is None: + profile = precomputed_speakers[voice_lower] + mode = str(profile.get("mode") or "xvec").lower() + params["speaker"] = [voice_lower] + params["task_type"] = ["Base"] + params["x_vector_only_mode"] = [mode != "icl"] + ref_text = request.ref_text or profile.get("ref_text") + if isinstance(ref_text, str) and ref_text.strip(): + params["ref_text"] = [ref_text] + ref_code_length = profile.get("ref_code_length") + if mode == "icl" and ref_code_length: + params["ref_code_length"] = [int(ref_code_length)] + logger.info("Using precomputed Qwen3-TTS custom voice profile: %s (mode=%s)", voice_lower, mode) + + elif params["task_type"][0] == "CustomVoice": + params["speaker"] = ["Vivian"] # Default for CustomVoice + + # Instructions for style/emotion control + if request.instructions is not None: + params["instruct"] = [request.instructions] + else: + params["instruct"] = [""] + + # Voice clone: ref_audio resolved in create_speech(), not here. + if request.ref_text is not None: + params["ref_text"] = [request.ref_text] + if request.speaker_embedding is not None: + # Store as plain float list (not tensor) so it survives msgspec + # serialization through the EngineCore IPC boundary. The talker's + # _build_prompt_embeds converts it back to a tensor on the GPU. + params["voice_clone_prompt"] = [ + { + "ref_spk_embedding": list(request.speaker_embedding), + } + ] + # speaker_embedding implies x_vector_only_mode + params["x_vector_only_mode"] = [True] + elif request.x_vector_only_mode is not None: + params["x_vector_only_mode"] = [request.x_vector_only_mode] + + # Generation parameters + if request.max_new_tokens is not None: + params["max_new_tokens"] = [request.max_new_tokens] + else: + params["max_new_tokens"] = [2048] + + if request.initial_codec_chunk_frames is not None: + params["initial_codec_chunk_frames"] = [request.initial_codec_chunk_frames] + + if request.non_streaming_mode is not None: + params["non_streaming_mode"] = [request.non_streaming_mode] + # Preserve the legacy VoiceDesign fallback when the request omits an + # explicit override. CustomVoice and Base rely on model defaults + # (True and False respectively). + elif params["task_type"][0] == "VoiceDesign": + params["non_streaming_mode"] = [True] + + return params + + def _estimate_prompt_len(self, tts_params: dict[str, Any]) -> int: + """Estimate prompt length so the placeholder matches model-side embeddings.""" + try: + from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import ( + Qwen3TTSPromptEmbedsBuilder, + ) + + server = self.ctx.server + if server._tts_tokenizer is None: + from transformers import AutoTokenizer + + model_name = self.ctx.engine_client.model_config.model + server._tts_tokenizer = AutoTokenizer.from_pretrained( + model_name, + trust_remote_code=True, + padding_side="left", + ) + hf_config = self.ctx.engine_client.model_config.hf_config + talker_config = hf_config.talker_config + task_type = (tts_params.get("task_type") or ["CustomVoice"])[0] + return Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information( + additional_information=tts_params, + task_type=task_type, + tokenize_prompt=lambda t: server._tts_tokenizer(t, padding=False)["input_ids"], + codec_language_id=getattr(talker_config, "codec_language_id", None), + spk_is_dialect=getattr(talker_config, "spk_is_dialect", None), + estimate_ref_code_len=self._estimate_ref_code_len, + ) + except Exception as e: + logger.warning("Failed to estimate TTS prompt length, using fallback 2048: %s", e) + return 2048 + + def _qwen3_tts_can_use_ref_audio_artifact_only(self, tts_params: dict[str, Any], artifact_key: str | None) -> bool: + server = self.ctx.server + x_vector_only = server._tts_x_vector_only(tts_params) + if not artifact_key or (artifact_key, x_vector_only) not in server._ref_audio_model_artifact_ready: + return False + return (tts_params.get("task_type") or ["CustomVoice"])[0] == "Base" + async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt, tts_params, warmup_key = await self.ctx.server._build_qwen3_tts_request(request) + """Build prompt + tts_params for Qwen3-TTS. + + Called from ``Qwen3TTSAdapter.build``. Returns + ``(prompt, tts_params, warmup_artifact_key)`` where the warmup key is the + Qwen3-TTS ref-audio artifact tracked after ``generate()``. + """ + server = self.ctx.server + qwen3_ref_audio_warmup_artifact_key: str | None = None + tts_params = self._build_tts_params(request) + # Resolve ref_audio (explicit or auto-set for uploaded voices) + # to [[wav_list, sr]] so the model doesn't re-decode base64. + ref_audio_source = request.ref_audio + if ref_audio_source is None and isinstance(tts_params.get("ref_audio"), list): + # Uploaded voice: ref_audio was auto-set as [base64_data_url] + ref_audio_source = tts_params["ref_audio"][0] + if ref_audio_source is not None and isinstance(ref_audio_source, str): + wav_list, sr, cache_key = await server._resolve_ref_audio(ref_audio_source) + tts_params["ref_audio_cache_key"] = cache_key + artifact_key = server._get_resolved_ref_audio_artifact_key(cache_key) + if artifact_key: + tts_params[_REF_AUDIO_CACHE_KEY] = [artifact_key] + ref_code_length = self._estimate_ref_code_len([wav_list, sr]) + if ref_code_length is not None: + tts_params["ref_code_length"] = [int(ref_code_length)] + if self._qwen3_tts_can_use_ref_audio_artifact_only(tts_params, artifact_key): + logger.debug("Using Qwen3-TTS ref_audio artifact-only path: %s", artifact_key) + else: + tts_params["ref_audio"] = [[wav_list, sr]] + qwen3_ref_audio_warmup_artifact_key = artifact_key + + ph_len = await self._estimate_prompt_len_async(tts_params) + prompt = tokens_input(prompt_token_ids=[1] * ph_len) + prompt["additional_information"] = tts_params + prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest( prompt=prompt, tts_params=tts_params, model_type=tts_params.get("task_type", ["unknown"])[0], - warmup_artifact_key=warmup_key, + warmup_artifact_key=qwen3_ref_audio_warmup_artifact_key, ) def _get_expected_speaker_embedding_dim(self) -> int: @@ -340,6 +565,16 @@ class Qwen3TTSAdapter(ARTTSAdapter): if isinstance(additional_information, dict) and effective_cap is not None: additional_information[QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY] = [effective_cap] + # Propagate the deploy/YAML stage seed to residual MTP sampling. The + # generic serving layer applies an explicit request.seed afterwards, + # so the request value still has higher precedence. + stage0_params = sampling_params_list[0] + default_seed = getattr(stage0_params, "seed", None) + if default_seed is not None: + if stage0_params.extra_args is None: + stage0_params.extra_args = {} + stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed)) + logger.debug( "Qwen3-TTS codec budget: task_type=%s text_tokens=%s dynamic_cap=%s " "configured_cap=%s request_cap=%s effective_cap=%s", diff --git a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py index 2c720d8e1..7ca5a44ff 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/step_audio2.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Step-Audio2 serving adapter.""" from typing import TYPE_CHECKING @@ -30,5 +31,17 @@ class StepAudio2Adapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = self.ctx.server._build_step_audio2_prompt(request) + """Build a chat prompt ending at ````. + + The assistant turn deliberately omits ``<|im_end|>`` so the thinker + continues by generating audio tokens. + """ + system_prompt = request.instructions or "You are a voice assistant. Read the text aloud." + prompt = { + "prompt": ( + f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + f"<|im_start|>user\n{request.input}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + } return PreparedRequest(prompt=prompt, tts_params={}, model_type="step_audio2") diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index 760bb3339..f4574004e 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -1,9 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """VoxCPM2 serving adapter (AR base-LM + diffusion side-computation).""" import time from typing import Any +import numpy as np from vllm.logger import init_logger from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -28,6 +30,63 @@ class VoxCPM2Adapter(ARTTSAdapter): # claims still resolves to VoxCPM2. detect_priority = 10 + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._tokenizer = None + self._split_map: dict[int, list[int]] = {} + + def _encode(self, text: str) -> list[int]: + from transformers import AutoTokenizer + + from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import ( + build_cjk_split_map, + split_multichar_chinese, + ) + + if self._tokenizer is None: + self._tokenizer = AutoTokenizer.from_pretrained( + self.ctx.engine_client.model_config.model, trust_remote_code=True + ) + self._split_map = build_cjk_split_map(self._tokenizer) + logger.info("VoxCPM2 serving: built multichar split map (%d entries)", len(self._split_map)) + ids = self._tokenizer.encode(text, add_special_tokens=True) + return split_multichar_chinese(ids, self._split_map) + + async def _build_prompt( + self, + request: "OpenAICreateSpeechRequest", + uploaded_ref: tuple[np.ndarray, int] | None = None, + ) -> dict[str, Any]: + """Build the full-length VoxCPM2 prefill prompt. + + ``uploaded_ref`` supplies an uploaded voice waveform when the request + has no explicit ``ref_audio`` so prefill-length accounting includes it. + """ + from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import build_voxcpm2_prompt + + server = self.ctx.server + self._encode("") + ref_audio = None + ref_sr = None + voice_profile = None + if request.ref_audio is not None: + ref_audio, ref_sr, _ = await server._resolve_ref_audio(request.ref_audio) + elif uploaded_ref is not None: + wav_np, ref_sr = uploaded_ref + ref_audio = wav_np.tolist() + elif request.voice is not None: + voice_profile = self.capabilities.precomputed_speakers.get(request.voice.lower()) + return build_voxcpm2_prompt( + hf_config=self.ctx.engine_client.model_config.hf_config, + tokenizer=self._tokenizer, + split_map=self._split_map, + text=request.input, + ref_audio=ref_audio, + ref_sr=ref_sr, + ref_text=request.ref_text, + voice_profile=voice_profile, + ) + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate VoxCPM2 request parameters. Returns error message or None.""" server = self.ctx.server @@ -66,7 +125,7 @@ class VoxCPM2Adapter(ARTTSAdapter): ) if request.ref_audio is None: uploaded_ref = server._load_uploaded_audio(voice_lower) - prompt = await server._build_voxcpm2_prompt(request, uploaded_ref=uploaded_ref) + prompt = await self._build_prompt(request, uploaded_ref=uploaded_ref) tts_params = {} if request.voice: voice_lower = request.voice.lower() diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index c9bea7bb0..f9d7639ca 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Voxtral TTS serving adapter.""" from typing import TYPE_CHECKING, Any +from vllm.utils.async_utils import make_async + from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers @@ -16,6 +19,39 @@ class VoxtralTTSAdapter(ARTTSAdapter): stage_keys = frozenset({"audio_generation"}) name = "voxtral_tts" + def __init__(self, ctx) -> None: + super().__init__(ctx) + self._build_prompt_async = make_async(self._build_prompt, executor=getattr(ctx.server, "_tts_executor", None)) + + def _build_prompt(self, request: "OpenAICreateSpeechRequest") -> dict[str, Any]: + """Build a Voxtral prompt for either a preset voice or inline reference audio.""" + from mistral_common.protocol.speech.request import SpeechRequest + from vllm.inputs import tokens_input + + ref_audio = request.ref_audio + if not request.voice and not ref_audio: + raise ValueError("Voxtral requires either a voice name or ref_audio.") + if isinstance(ref_audio, str) and ref_audio.startswith("data:"): + _, _, ref_audio = ref_audio.partition(",") + server = self.ctx.server + if server._tts_tokenizer is None: + from vllm.tokenizers import cached_tokenizer_from_config + + server._tts_tokenizer = cached_tokenizer_from_config(self.ctx.engine_client.model_config).instruct + if request.voice is not None: + tokenized = server._tts_tokenizer.encode_speech_request( + SpeechRequest(input=request.input, voice=request.voice) + ) + prompt = tokens_input(prompt_token_ids=tokenized.tokens) + prompt["additional_information"] = {"voice": [request.voice]} + return prompt + tokenized = server._tts_tokenizer.encode_speech_request(SpeechRequest(input=request.input, ref_audio=ref_audio)) + audio = tokenized.audios[0] + return { + "prompt_token_ids": tokenized.tokens, + "multi_modal_data": {"audio": [(audio.audio_array, audio.sampling_rate)]}, + } + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate Voxtral TTS request parameters. Returns error message or None.""" server = self.ctx.server @@ -48,7 +84,7 @@ class VoxtralTTSAdapter(ARTTSAdapter): async def build( self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: - prompt = await self.ctx.server._build_voxtral_prompt_async(request) + prompt = await self._build_prompt_async(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="voxtral_tts") def _load_supported_speakers(self) -> set[str]: diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index f8dc962f3..5febb8912 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """VoxCPM2 AR talker — PagedAttention pipeline with per-request state. Architecture: @@ -158,7 +158,7 @@ def build_voxcpm2_prompt( """Build a VoxCPM2 prefill prompt whose ``prompt_token_ids`` length matches the talker-side prefill length. - Used by both online serving (``serving_speech._build_voxcpm2_prompt``) and + Used by both online serving (``VoxCPM2Adapter._build_prompt``) and the offline example, so the talker-side length assertion never fires. """ ids = split_multichar_chinese(tokenizer.encode(text, add_special_tokens=True), split_map) @@ -2208,7 +2208,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): assert scaffold_len == tts_len, ( f"voxcpm2 prefill length mismatch: scaffold_len={scaffold_len} tts_len={tts_len}; " "caller must pad prompt_token_ids to the full prefill length " - "(see serving_speech._build_voxcpm2_prompt or the offline example)." + "(see VoxCPM2Adapter._build_prompt or the offline example)." ) enc_out = base_lm_out.unsqueeze(0)