From 8231f5c978aa432ebb53d408ab0a9b6e16fcee8b Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Tue, 8 Sep 2026 16:00:43 +0800 Subject: [PATCH] [Bugfix] Use reference transcripts for MOSS-TTS Local continuation Build the official assistant audio prefix when Local cloning requests include a reference transcript. Preserve the audio-only fallback, conditioning cache identity, seed and output budget. Cover the prompt contract and fallback in the existing adapter tests. Signed-off-by: Sy03 <1370724210@qq.com> --- recipes/OpenMOSS/MOSS-TTS.md | 12 ++++- .../openai_api/test_tts_adapter.py | 51 +++++++++++++++++++ .../openai/tts_adapters/moss_tts.py | 39 ++++++++++++-- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/recipes/OpenMOSS/MOSS-TTS.md b/recipes/OpenMOSS/MOSS-TTS.md index e9cb798fe..8d37039d8 100644 --- a/recipes/OpenMOSS/MOSS-TTS.md +++ b/recipes/OpenMOSS/MOSS-TTS.md @@ -16,7 +16,7 @@ Use this recipe for 24 kHz multilingual TTS with voice cloning (20 languages inc Chinese and English). Choose a variant based on your latency and quality requirements: | Model | Params | Use case | -|---|---|---| +| --- | --- | --- | | MOSS-TTS | 8B | General TTS, highest quality | | MOSS-TTS-v1.5 | 8B | General TTS upgrade of 1.0: 31 languages, steadier cloning, `[pause Xs]` markers (set `language` for best results); same `MossTTSDelay` API | | MOSS-TTS-Realtime | 1.7B | Lowest latency (TTFB ~180 ms), streaming-first | @@ -24,9 +24,17 @@ Chinese and English). Choose a variant based on your latency and quality require | MOSS-SoundEffect | 8B | Sound effect synthesis from text description | | MOSS-VoiceGenerator | 1.7B | Zero-shot voice design | -All variants share the same codec (`OpenMOSS-Team/MOSS-Audio-Tokenizer`, ~7 GB) and +The variants above share the same codec (`OpenMOSS-Team/MOSS-Audio-Tokenizer`, ~7 GB) and output 24 kHz mono audio. +MOSS-TTS-Local-Transformer-v1.5 uses MOSS-Audio-Tokenizer-v2 and outputs 48 kHz +stereo audio. For Local voice cloning through `/v1/audio/speech`, provide an +accurate `ref_text` transcript alongside `ref_audio`. The adapter uses the +reference transcript and audio as a continuation prefix, then generates only +the requested `input` speech. Without a nonblank `ref_text`, Local uses +audio-reference generation. Reference transcripts must match the reference +audio; they are not style instructions. + ## References - Offline inference example: [`examples/offline_inference/text_to_speech/moss_tts/`](../../examples/offline_inference/text_to_speech/moss_tts/) diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index d1a0174c3..ed71167a3 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -162,6 +162,57 @@ def test_moss_tts_seed_falls_back_to_stage_default(adapter_cls, mocker): assert prepared.tts_params["seed"] == [42] +@pytest.mark.parametrize("variant,ref_text", [("local", None), ("local", " "), ("tts", "Reference.")]) +def test_moss_tts_generation_fallback(variant, ref_text, mocker): + server = mocker.Mock(_moss_variant=variant) + server._build_moss_tts_params = mocker.AsyncMock(return_value={"prompt_token_ids": [7]}) + request = OpenAICreateSpeechRequest(input="Target.", ref_text=ref_text) + adapter = MossTTSAdapter(SpeechServingContext(server=server)) + + prepared = asyncio.run(adapter.build(request, [], has_inline_ref_audio=True)) + + server._build_moss_tts_params.assert_awaited_once_with(request, has_inline_ref_audio=True) + server._get_moss_processor.assert_not_called() + assert prepared.prompt["prompt_token_ids"] == [7] + + +def test_moss_local_reference_transcript_continuation(mocker): + import torch + + reference = [torch.ones((3, 12), dtype=torch.int64)] + unified = torch.arange(52, dtype=torch.int64).reshape(1, 4, 13) + processor = mocker.Mock(return_value={"input_ids": unified}) + server = mocker.Mock(_moss_variant="local", uploaded_speakers={"speaker": {}}) + server._build_moss_tts_params = mocker.AsyncMock(return_value={"prompt_token_ids": [7]}) + server._get_moss_processor.return_value = processor + server._voice_created_at.return_value = 123 + server._encode_moss_references = mocker.AsyncMock(return_value=(reference, {0: "reference-key"})) + request = OpenAICreateSpeechRequest( + input="Target.", ref_text=" Reference. ", language="English", voice="Speaker", seed=0, max_new_tokens=2048 + ) + adapter = MossTTSAdapter(SpeechServingContext(server=server)) + + prepared = asyncio.run(adapter.build(request, [], has_inline_ref_audio=False)) + + processor.build_user_message.assert_called_once_with(text="Reference. Target.", language="English") + processor.build_assistant_message.assert_called_once_with(audio_codes_list=reference) + processor.assert_called_once_with( + conversations=[[processor.build_user_message.return_value, processor.build_assistant_message.return_value]], + mode="continuation", + ) + server._encode_moss_references.assert_awaited_once_with(request, has_inline_ref_audio=False, two_speaker=False) + server._build_moss_tts_params.assert_not_called() + assert prepared.prompt["prompt_token_ids"] == unified[0, :, 0].tolist() + assert torch.equal(prepared.tts_params["codes"]["ref"], unified[0, :, 1:]) + assert prepared.tts_params["max_new_frames"] == [2048] + assert prepared.tts_params["ref_audio_cache_key"] == "reference-key" + assert prepared.tts_params["voice_name"] == ["speaker"] + assert prepared.tts_params["voice_created_at"] == [123] + assert prepared.tts_params["seed"] == [0] + assert prepared.prompt["cache_salt"] + assert request.input == "Target." + + def test_qwen3_tts_metadata(): assert Qwen3TTSAdapter.backend == "ar" assert issubclass(Qwen3TTSAdapter, ARTTSAdapter) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 3d6e99552..8f3ee8c08 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -2,8 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """MOSS-TTS serving adapters (Nano + full family). -Both variants share the same build/validate flow (``_build_moss_tts_params`` -handles each); they are registered under distinct model-type names. +Variants share validation and request metadata. Local uses transcript-guided +continuation when the request includes a reference transcript. """ from typing import TYPE_CHECKING @@ -23,6 +23,36 @@ if TYPE_CHECKING: class _MossTTSAdapterBase(ARTTSAdapter): + async def _build_local_continuation_params( + self, request: "OpenAICreateSpeechRequest", *, has_inline_ref_audio: bool + ) -> dict: + import torch + + server = self.ctx.server + processor = server._get_moss_processor() + reference, resolve_keys = await server._encode_moss_references( + request, has_inline_ref_audio=has_inline_ref_audio, two_speaker=False + ) + assert request.ref_text is not None + user_kwargs = {"text": request.ref_text.strip() + " " + (request.input or "")} + if request.language: + user_kwargs["language"] = request.language + # Align the reference transcript with an assistant audio prefix. Only + # newly generated frames go to the codec; the prefix is conditioning. + user = processor.build_user_message(**user_kwargs) + assistant = processor.build_assistant_message(audio_codes_list=reference) + batch = processor(conversations=[[user, assistant]], mode="continuation") + unified = batch["input_ids"][0] + params = { + "prompt_token_ids": unified[:, 0].tolist(), + "codes": {"ref": unified[:, 1:].contiguous().to(torch.int64)}, + } + if request.max_new_tokens is not None: + params["max_new_frames"] = [request.max_new_tokens] + if 0 in resolve_keys: + params["ref_audio_cache_key"] = resolve_keys[0] + return params + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: """Validate any MOSS-TTS-family request (nano + 5 full variants). @@ -92,7 +122,10 @@ class _MossTTSAdapterBase(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: server = self.ctx.server - tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio) + if server._moss_variant == "local" and request.ref_text and request.ref_text.strip(): + tts_params = await self._build_local_continuation_params(request, has_inline_ref_audio=has_inline_ref_audio) + else: + tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio) if request.voice: voice_lower = request.voice.lower() if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: