diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 39980c6e6..5d0ca6dbc 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -68,7 +68,7 @@ th { | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — | | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — | | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — | -| `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — | +| `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — | | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — | | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — | | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — | diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index 92e08fab5..c21bff5fe 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -24,7 +24,7 @@ list of supported architectures across all modalities, see | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz | | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz | -| dots.tts | `rednote-hilab/dots.tts-soar` | single (native AR) | — (not wired yet) | — | — | 48 kHz | +| dots.tts | `dots-studio/dots.tts-soar` | single (native AR) | — (text-only) | ✓ (online) | no-reference synthesis | 48 kHz | | IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz | | IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz | @@ -490,20 +490,20 @@ Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B ba ### Quick start ```bash python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ - --model rednote-hilab/dots.tts-soar \ + --model dots-studio/dots.tts-soar \ --text "Hello, this is a test of dots TTS running on vLLM Omni." ``` ### Voice cloning -Not wired in this release — generation is zero-shot only. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet. +Not wired in this release. Generation is text-only and does not consume reference audio, a named voice, or a precomputed speaker embedding. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet. ### Streaming -The AudioVAE decoder has an internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`) used to avoid boundary artifacts between 160 ms patches, but it is not yet exposed through an online serving endpoint or example. +The AudioVAE decoder emits incremental audio through its internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`). Online streaming is exposed through the OpenAI-compatible `/v1/audio/speech` endpoint with `stream=true`; see the [online serving guide](../../online_serving/text_to_speech/README.md#dotstts). The offline `end2end.py` script returns the consolidated waveform. ### Notes - Output: 48 kHz mono WAV. - Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`). -- Checkpoints: `rednote-hilab/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet. +- Checkpoints: `dots-studio/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet. - Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap. --- diff --git a/examples/offline_inference/text_to_speech/dots_tts/end2end.py b/examples/offline_inference/text_to_speech/dots_tts/end2end.py index 8705f03f3..aa710db4a 100644 --- a/examples/offline_inference/text_to_speech/dots_tts/end2end.py +++ b/examples/offline_inference/text_to_speech/dots_tts/end2end.py @@ -1,7 +1,7 @@ """Offline dots.tts inference example (single-stage native AR pipeline). Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml). -Reference checkpoint: rednote-hilab/dots.tts-soar. +Reference checkpoint: dots-studio/dots.tts-soar. """ from __future__ import annotations @@ -25,7 +25,7 @@ def parse_args(): parser.add_argument( "--model", type=str, - default="rednote-hilab/dots.tts-soar", + default="dots-studio/dots.tts-soar", help="dots.tts model path or HuggingFace repo ID.", ) parser.add_argument( diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 500e52773..e1cdd58dd 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -24,6 +24,7 @@ For the full list of supported architectures across all modalities, see | OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) | | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ | +| dots.tts | `dots-studio/dots.tts-soar` | — (text-only) | ✓ (PCM stream) | — (default placeholder only) | — | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ | CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead. @@ -99,6 +100,44 @@ For full request-shape documentation (all parameters, response formats, error co --- +## dots.tts + +Single-stage, text-only TTS at 48 kHz. The current integration supports +no-reference synthesis; voice cloning, reference audio, named voices, and +precomputed speaker embeddings are not supported yet. + +### Launch + +```bash +vllm serve dots-studio/dots.tts-soar --omni --trust-remote-code --port 8091 +``` + +### Sending requests + +```bash +# Non-streaming WAV +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is a dots TTS online-serving test.", + "voice": "default", + "response_format": "wav" + }' --output output.wav + +# Streaming 48 kHz mono PCM +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is a streaming dots TTS test.", + "voice": "default", + "stream": true, + "stream_format": "audio", + "response_format": "pcm" + }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 - +``` + +--- + ## GLM-TTS 2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`. diff --git a/recipes/rednote-hilab/dots.tts.md b/recipes/rednote-hilab/dots.tts.md index d5c7868cb..cc180020e 100644 --- a/recipes/rednote-hilab/dots.tts.md +++ b/recipes/rednote-hilab/dots.tts.md @@ -1,20 +1,19 @@ # dots.tts -> Offline continuous-AR TTS at 48 kHz (rednote-hilab) +> Continuous-AR TTS at 48 kHz (rednote-hilab) ## Summary - Vendor: rednote-hilab -- Model: `rednote-hilab/dots.tts-soar` +- Model: `dots-studio/dots.tts-soar` - Task: Text-to-speech, zero-shot synthesis only -- Mode: Offline end-to-end example only (not yet wired for online serving — - see [Known limitations](#known-limitations)) +- Mode: Offline inference and OpenAI-compatible online serving - Maintainer: Community ## When to use this recipe Use this recipe as a known-good starting point for running -`rednote-hilab/dots.tts-soar` offline on vLLM-Omni on consumer-class GPUs. +`dots-studio/dots.tts-soar` on vLLM-Omni on consumer-class GPUs. dots.tts is a ~1.7B-parameter continuous-AR TTS model (Qwen2.5-1.5B base LM + 344M DiT flow-matching head + 180M AudioVAE) that emits 48 kHz mono audio. It follows the same "vLLM-native base LM + side-path computation" pattern as @@ -23,8 +22,8 @@ VoxCPM2 — single-stage pipeline loopback → AudioVAE (streaming decode)` — with a plain Qwen2 backbone instead of MiniCPM4, and no FSQ / residual-LM stage. -This is an early integration (see [Known limitations](#known-limitations) -below before depending on it for anything beyond offline experimentation). +This is an early integration; review the [Known limitations](#known-limitations) +before deploying it in production. ## References @@ -32,6 +31,8 @@ below before depending on it for anything beyond offline experimentation). [`examples/offline_inference/text_to_speech/dots_tts/end2end.py`](../../examples/offline_inference/text_to_speech/dots_tts/end2end.py) - Example guide: [`examples/offline_inference/text_to_speech/README.md`](../../examples/offline_inference/text_to_speech/README.md#dotstts) +- Online serving guide: + [`examples/online_serving/text_to_speech/README.md`](../../examples/online_serving/text_to_speech/README.md#dotstts) - Default deploy config: [`vllm_omni/deploy/dots_tts.yaml`](../../vllm_omni/deploy/dots_tts.yaml) - Talker / pipeline source: @@ -73,7 +74,7 @@ release runs `enforce_eager: true`, so it doesn't apply today). ```bash python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ - --model rednote-hilab/dots.tts-soar \ + --model dots-studio/dots.tts-soar \ --text "Hello, this is a test of dots TTS running on vLLM Omni." ``` @@ -88,7 +89,7 @@ Pass `--deploy-config ` to override. ```bash python examples/offline_inference/text_to_speech/dots_tts/end2end.py \ - --model rednote-hilab/dots.tts-soar \ + --model dots-studio/dots.tts-soar \ --text "Hello, this is a test of dots TTS running on vLLM Omni." ``` @@ -106,10 +107,29 @@ Whisper transcription of the output matched the input text with no dropped leading word (confirms the streaming-vocoder patch-boundary fix described in [Known limitations](#known-limitations)). +**T2 — online text-only synthesis**: + +```bash +vllm serve dots-studio/dots.tts-soar --omni --trust-remote-code --port 8091 +``` + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is a test of dots TTS online serving.", + "voice": "default", + "response_format": "wav" + }' --output output.wav +``` + +The endpoint also supports raw streaming audio with `stream=true`, +`stream_format="audio"`, and `response_format="pcm"`. + #### Notes - Output: 48 kHz mono WAV. -- Checkpoints: `rednote-hilab/dots.tts-soar` is the validated default +- Checkpoints: `dots-studio/dots.tts-soar` is the validated default used throughout this recipe. `rednote-hilab/dots.tts-base` shares the same architecture but is unvalidated in this repo. `rednote-hilab/dots.tts-mf` (MeanFlow, 2-4 step) is not supported — see below. @@ -123,17 +143,11 @@ described in [Known limitations](#known-limitations)). ## Known limitations -- **No online serving yet.** dots.tts is not registered in - `vllm_omni/entrypoints/openai/tts_adapters/` (the framework other TTS - models in this repo use for `/v1/audio/speech`), so `vllm serve - rednote-hilab/dots.tts-soar --omni` will start but the OpenAI-compatible - speech endpoint does not know how to build request params for it yet. - Offline `Omni()` (as used in `end2end.py`) is the only supported path - today. - **Voice cloning is not wired.** The CAM++ x-vector speaker encoder weights load and are exercised by `load_weights()`, but nothing in the - prompt builder consumes a reference audio yet — generation is zero-shot - only. `end2end.py` has no `--ref-audio`/`--ref-text` flags. + prompt builder consumes reference audio or a precomputed speaker embedding + yet. Both offline and online generation are text-only; `ref_audio`, + `ref_text`, named voices, and speaker embeddings are not supported. - **`dots.tts-mf` (MeanFlow, 2-4 step) checkpoint is not supported.** Only the fixed 10-step Euler DiT sampler used by `dots.tts-soar` / `dots.tts-base` is implemented. diff --git a/tests/e2e/online_serving/test_dots_tts_expansion.py b/tests/e2e/online_serving/test_dots_tts_expansion.py new file mode 100644 index 000000000..34bc54b8e --- /dev/null +++ b/tests/e2e/online_serving/test_dots_tts_expansion.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E Online expansion tests for dots.tts(text-only). +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.full_model, pytest.mark.tts] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "dots-studio/dots.tts-soar" +DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0 +_MIN_AUDIO_BYTES = 40_000 +MAX_CONCURRENT = 4 + + +def get_prompt(prompt_type="text"): + prompts = { + "text": "The weather is nice today, perfect for a walk in the park.", + } + return prompts.get(prompt_type, prompts["text"]) + + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_deploy_config_path("dots_tts.yaml"), + server_args=["--trust-remote-code", "--disable-log-stats"], + ), + id="dots_tts", + ) +] + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_only_streaming_001(omni_server, openai_client) -> None: + """ + Test text-only text-to-audio via OpenAI API. + Deploy Setting: dots_tts.yaml + Input Modal: text + Output Modal: audio + Input Setting: stream=True + Datasets: few requests (max_num_seqs=4) + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": True, + "stream_format": "audio", + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "response_format": "wav", + "voice": "default", + "min_audio_bytes": _MIN_AUDIO_BYTES, + } + openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) + + +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_only_nonstreaming_pcm_001(omni_server, openai_client) -> None: + """ + Test text-only non-stream PCM output via OpenAI API. + Deploy Setting: dots_tts.yaml + Input Modal: text + Output Modal: audio (pcm) + Input Setting: stream=False + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "response_format": "pcm", + "stream": False, + "timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S, + "voice": "default", + "min_audio_bytes": _MIN_AUDIO_BYTES, + "min_hnr_db": -2.0, + } + openai_client.send_audio_speech_request(request_config) diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 0af78787b..6b8fff0a5 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -4775,6 +4775,89 @@ class TestMingFlashOmniTTSServing: ming_flash_omni_tts_server._build_ming_flash_omni_prompt.assert_called_once() +@pytest.fixture +def dots_tts_server(mocker: MockerFixture): + mocker.patch( + "vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers", + return_value=set(), + ) + mocker.patch( + "vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate", + return_value=None, + ) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock( + model="dots-studio/dots.tts-soar", + ) + mock_engine_client.default_sampling_params_list = [ + SimpleNamespace(max_tokens=2048, min_tokens=None, extra_args=None) + ] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace(model_stage="latent_generator", model_arch="DotsTTSForConditionalGeneration"), + tts_args={}, + ) + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + + +class TestDotsTTSServing: + def test_dots_tts_prompt_validation(self, dots_tts_server): + request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript") + error = dots_tts_server._validate_tts_request(request) + assert error is not None + assert "ref_text" in error + + request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc") + error = dots_tts_server._validate_tts_request(request) + assert error is not None + assert "ref_audio" in error + + request = OpenAICreateSpeechRequest(input="Hello", voice="test") + error = dots_tts_server._validate_tts_request(request) + assert error is not None + assert "voice" in error + + request = OpenAICreateSpeechRequest(input="Hello", speaker_embedding=[1, 2, 3]) + error = dots_tts_server._validate_tts_request(request) + assert error is not None + assert "speaker_embedding" in error + + request = OpenAICreateSpeechRequest(input="Hello", x_vector_only_mode=True) + error = dots_tts_server._validate_tts_request(request) + assert error is not None + assert "x_vector_only_mode" in error + + def test_dots_tts_adapter_awaits_async_prompt_builder(self, dots_tts_server, mocker: MockerFixture): + build_prompt_async = mocker.patch.object( + dots_tts_server._adapter, + "_build_prompt_async", + new=mocker.AsyncMock(return_value={"prompt_token_ids": [1, 2, 3]}), + ) + request = OpenAICreateSpeechRequest(input="Hello") + asyncio.run(dots_tts_server._prepare_speech_generation(request)) + build_prompt_async.assert_awaited_once_with("Hello") + + def test_dots_tts_adapter_apply_sampling_overrides(self, dots_tts_server, mocker: MockerFixture): + mocker.patch.object(dots_tts_server._adapter, "build", return_value=PreparedRequest(prompt="Hello")) + request = OpenAICreateSpeechRequest(input="Hello", max_new_tokens=10) + asyncio.run(dots_tts_server._prepare_speech_generation(request)) + sampling_params_list = dots_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"] + assert sampling_params_list[0].max_tokens == 10 + + class TestTTSAsyncOffloading: """Tests for event-loop-safe offloading of blocking TTS operations.""" diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 03faacfcc..d1a0174c3 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -64,6 +64,7 @@ EXPECTED_MODEL_TYPES = { "step_audio2", "indextts2", "indextts2_5", + "dots_tts", } diff --git a/tests/entrypoints/openai_api/test_tts_detection.py b/tests/entrypoints/openai_api/test_tts_detection.py index 6b8b97849..3b8d17997 100644 --- a/tests/entrypoints/openai_api/test_tts_detection.py +++ b/tests/entrypoints/openai_api/test_tts_detection.py @@ -255,6 +255,11 @@ def test_arch_matching_is_a_fallback_not_an_override(): assert detect_tts_model_type("ming_tts", "MingTTSForConditionalGeneration") == "ming_flash_omni_tts" +def test_shared_latent_generator_resolves_by_architecture_priority(): + assert detect_tts_model_type("latent_generator", "VoxCPM2TalkerForConditionalGeneration") == "voxcpm2" + assert detect_tts_model_type("latent_generator", "DotsTTSForConditionalGeneration") == "dots_tts" + + def test_stage_keys_cover_legacy_stage_set(): """No stage key was dropped when the module constants were deleted.""" assert _LEGACY_TTS_MODEL_STAGES <= all_tts_stage_keys() diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index dcf8c4eaa..a09d97b68 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -106,6 +106,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 audex_tta, cosyvoice3, covo_audio, + dots_tts, fish_speech, glm_tts, higgs_audio_v2, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/dots_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/dots_tts.py new file mode 100644 index 000000000..75cfb8455 --- /dev/null +++ b/vllm_omni/entrypoints/openai/tts_adapters/dots_tts.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +"""dots.tts serving adapter.""" + +from typing import TYPE_CHECKING + +from transformers import AutoTokenizer +from vllm.utils.async_utils import make_async + +from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens + +if TYPE_CHECKING: + from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest + + +@register_tts_adapter +class DotsTTSAdapter(ARTTSAdapter): + """Adapter for dots.tts (AR ``engine_client`` backend). + + The current integration supports text-only, no-reference speech synthesis. + Voice cloning and reference-audio conditioning are not supported yet. + """ + + stage_keys = frozenset() + model_archs = frozenset({"DotsTTSForConditionalGeneration"}) + name = "dots_tts" + detect_priority = 5 + + def __init__(self, ctx): + super().__init__(ctx) + self.tokenizer = None + self._build_prompt_async = None + + def _build_prompt(self, text: str) -> dict: + from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import build_dots_tts_prompt + + if self.tokenizer is None: + self.tokenizer = AutoTokenizer.from_pretrained( + self.ctx.engine_client.model_config.model, + trust_remote_code=True, + ) + return build_dots_tts_prompt(self.tokenizer, text) + + def validate(self, request: "OpenAICreateSpeechRequest") -> str | None: + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + if request.voice not in (None, "default"): + return "'voice' is not supported for dots.tts" + + if request.ref_audio is not None: + return "'ref_audio' is not supported for dots.tts" + + if request.ref_text is not None: + return "'ref_text' is not supported for dots.tts" + + if request.speaker_embedding is not None: + return "'speaker_embedding' is not supported for dots.tts" + + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is not supported for dots.tts" + return None + + async def build( + self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool + ) -> PreparedRequest: + if self._build_prompt_async is None: + self._build_prompt_async = make_async( + self._build_prompt, + executor=self.ctx.server._tts_executor, + ) + prompt = await self._build_prompt_async(request.input) + return PreparedRequest(prompt=prompt, tts_params={}, model_type="dots_tts") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py b/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py index 368a3fd1e..ff2f460d6 100644 --- a/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py +++ b/vllm_omni/model_executor/models/dots_tts/dots_tts_talker.py @@ -77,7 +77,7 @@ def _require_config_block(hf_config: Any, name: str) -> dict[str, Any]: raise ValueError( f"dots.tts checkpoint config.json has no '{name}' block — cannot " "build the audio side path. Expected the upstream dots.tts config " - "layout (e.g. rednote-hilab/dots.tts-soar)." + "layout (e.g. dots-studio/dots.tts-soar)." ) return block @@ -189,7 +189,7 @@ class _RequestState: vocoder_stream_state: Any | None = None -# DiT transformer hyperparameters from rednote-hilab/dots.tts-soar config.json +# DiT transformer hyperparameters from dots-studio/dots.tts-soar config.json # (the ``DiT`` block). Mirrors the shape of an upstream pydantic config: DiT's # constructor calls ``transformer_config.to_dict()`` and reads ``.hidden_size`` # / ``.num_layers``, so this @dataclass plus the ``to_dict`` helper satisfies @@ -226,7 +226,7 @@ def _build_dit_config(hf_config: Any) -> _DiTConfig: return _DiTConfig(**_require_config_block(hf_config, "DiT")) -# Patch encoder hyperparameters from rednote-hilab/dots.tts-soar config.json +# Patch encoder hyperparameters from dots-studio/dots.tts-soar config.json # (the top-level ``patch_size`` and the ``PatchEncoder`` sub-block). # # Two layers of config: @@ -505,7 +505,7 @@ class DotsTTSForConditionalGeneration(nn.Module): """Resolve latent_stats.pt to a local filesystem path. ``vllm_config.model_config.model`` is typically an HF repo ID like - ``"rednote-hilab/dots.tts-soar"`` — joining it with ``"latent_ + ``"dots-studio/dots.tts-soar"`` — joining it with ``"latent_ stats.pt"`` gives a non-existent path. Try the local-dir form first (covers ``--model /local/path``), then fall back to HF cache lookup with ``local_files_only=True`` so we never trigger diff --git a/vllm_omni/transformers_utils/configs/dots_tts.py b/vllm_omni/transformers_utils/configs/dots_tts.py index 1950a2b3a..185d23ad7 100644 --- a/vllm_omni/transformers_utils/configs/dots_tts.py +++ b/vllm_omni/transformers_utils/configs/dots_tts.py @@ -22,7 +22,7 @@ Future work: read ``llm_config.json`` during ``from_pretrained`` so a checkpoint with a different LM shape works without editing this file. Upstream reference: - https://huggingface.co/rednote-hilab/dots.tts-soar/blob/main/llm_config.json + https://huggingface.co/dots-studio/dots.tts-soar/blob/main/llm_config.json """ from __future__ import annotations