diff --git a/docs/user_guide/examples/offline_inference/moss_tts_nano.md b/docs/user_guide/examples/offline_inference/moss_tts_nano.md new file mode 100644 index 000000000..21830c258 --- /dev/null +++ b/docs/user_guide/examples/offline_inference/moss_tts_nano.md @@ -0,0 +1,79 @@ +# MOSS-TTS-Nano + +Source . + + +Single-stage offline TTS pipeline using the 0.1B MOSS-TTS-Nano AR LM and MOSS-Audio-Tokenizer-Nano codec. Outputs 48 kHz stereo WAV. + +## Quick Start + +```bash +python end2end.py --text "Hello, this is MOSS-TTS-Nano." +``` + +The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face. + +## Usage + +``` +python end2end.py [OPTIONS] + +Options: + --text TEXT Text to synthesize (default: "Hello, this is MOSS-TTS-Nano speaking.") + --voice VOICE Built-in voice preset (default: Junhao) + --mode MODE continuation (default) or voice_clone + --prompt-audio PATH Reference WAV/MP3 for custom voice cloning + --prompt-text TEXT Reference transcript (continuation mode) + --max-new-frames N Max AR frames, default 375 (~14 s audio) + --seed INT Random seed for reproducibility + --batch Run a built-in batch of diverse samples (ZH/EN) + --output-dir DIR Directory for WAV outputs (default: /tmp/moss_tts_nano_output) + --stage-config PATH Path to stage config YAML +``` + +## Examples + +```bash +# Built-in Chinese voice +python end2end.py --text "你好，这是MOSS-TTS-Nano的语音合成演示。" --voice Junhao + +# Built-in English voice +python end2end.py --text "Hello from MOSS-TTS-Nano." --voice Ava + +# Batch synthesis +python end2end.py --batch --output-dir /tmp/batch_output + +# Reproducible output +python end2end.py --text "Deterministic test." --seed 42 +``` + +## Built-in Voice Presets + +| Voice | Language | +|-------|----------| +| `Junhao` | ZH | +| `Zhiming` | ZH | +| `Weiguo` | ZH | +| `Xiaoyu` | ZH | +| `Yuewen` | ZH | +| `Lingyu` | ZH | +| `Ava` | EN | +| `Bella` | EN | +| `Adam` | EN | +| `Nathan` | EN | +| `Sakura` | JA | +| `Yui` | JA | +| `Aoi` | JA | +| `Hina` | JA | +| `Mei` | JA | + +## Stage Config + +The stage config is at `vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml`. + +```yaml +engine_args: + gpu_memory_utilization: 0.3 # ~2 GB VRAM + max_num_seqs: 4 # concurrent requests + max_model_len: 4096 +``` diff --git a/docs/user_guide/examples/online_serving/moss_tts_nano.md b/docs/user_guide/examples/online_serving/moss_tts_nano.md new file mode 100644 index 000000000..1452c44cb --- /dev/null +++ b/docs/user_guide/examples/online_serving/moss_tts_nano.md @@ -0,0 +1,167 @@ +# MOSS-TTS-Nano + +Source . + + +## Installation + +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) + +## Supported Models + +| Model | Parameters | Description | +|-------|-----------|-------------| +| `OpenMOSS-Team/MOSS-TTS-Nano` | 0.1B | AR LM + MOSS-Audio-Tokenizer-Nano codec, 48 kHz stereo, ZH/EN/JA | + +## Gradio Demo + +!!! note "Gradio is an optional dependency" + The Gradio demo requires the `[demo]` extras. Install them first: + + ```bash + pip install 'vllm-omni[demo]' + ``` + +An interactive Gradio demo is available with multilingual voice presets, custom voice cloning, and gapless AudioWorklet streaming. + +```bash +# Option 1: Launch server + Gradio together +./run_gradio_demo.sh + +# Option 2: If server is already running +python gradio_demo.py --api-base http://localhost:8091 +``` + +Then open http://localhost:7860 in your browser. + +Features: + +- 15 built-in voice presets (6 ZH, 4 EN, 5 JA) +- Custom voice cloning from uploaded reference audio +- Gapless AudioWorklet streaming with TTFP/RTF metrics +- Non-streaming mode with WAV/MP3/FLAC download + +## Run examples (MOSS-TTS-Nano) + +### Launch the Server + +```bash +vllm-omni serve OpenMOSS-Team/MOSS-TTS-Nano \ + --stage-configs-path vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager +``` + +Or use the convenience script: + +```bash +./run_server.sh +``` + +### Using curl + +```bash +# Built-in voice preset (non-streaming) +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is MOSS-TTS-Nano.", + "voice": "Ava", + "response_format": "wav" + }' --output output.wav + +# Chinese voice +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "你好，这是语音合成测试。", + "voice": "Junhao", + "response_format": "wav" + }' --output output_zh.wav + +# Streaming (raw PCM) +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, streaming output from MOSS-TTS-Nano.", + "voice": "Ava", + "stream": true, + "response_format": "pcm" + }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 2 - +``` + +### Using Python + +```python +import httpx + +response = httpx.post( + "http://localhost:8091/v1/audio/speech", + json={ + "input": "Hello, this is MOSS-TTS-Nano.", + "voice": "Ava", + "response_format": "wav", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +## Built-in Voice Presets + +| Voice | Language | Description | +|-------|----------|-------------| +| `Junhao` | ZH | Male, standard Mandarin | +| `Zhiming` | ZH | Male | +| `Weiguo` | ZH | Male | +| `Xiaoyu` | ZH | Female | +| `Yuewen` | ZH | Female | +| `Lingyu` | ZH | Female | +| `Ava` | EN | Female, American English | +| `Bella` | EN | Female | +| `Adam` | EN | Male | +| `Nathan` | EN | Male | +| `Sakura` | JA | Female | +| `Yui` | JA | Female | +| `Aoi` | JA | Female | +| `Hina` | JA | Female | +| `Mei` | JA | Female | + +## API Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | **required** | Text to synthesize (ZH / EN / JA) | +| `voice` | string | `"Junhao"` | Built-in voice name (see table above) | +| `response_format` | string | `"wav"` | Audio format: wav, mp3, flac, pcm | +| `stream` | bool | false | Stream raw PCM chunks | + +## Output Format + +WAV files: 48 kHz mono. PCM streaming: 48 kHz stereo (2-channel interleaved int16). + +## Stage Config + +The stage config is at `vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml`. Key settings: + +```yaml +engine_args: + gpu_memory_utilization: 0.3 # ~2 GB VRAM; increase for faster init + max_num_seqs: 4 # concurrent requests + max_model_len: 4096 +``` + +## Troubleshooting + +1. **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ defaults to torchcodec which requires NVRTC. vLLM-Omni patches this automatically at model load time to use soundfile instead. +2. **Connection refused**: Ensure the server is running on the correct port. +3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings. +4. **Out of memory**: The default `gpu_memory_utilization=0.3` is conservative. Increase it in the stage config if you have more VRAM available. + +## Known Limitations + +- **Streaming TTFP**: Single-stage generation models currently run the full inference before returning audio. True progressive streaming (low TTFP) requires multi-step scheduling support in GPUGenerationModelRunner. diff --git a/tests/e2e/online_serving/test_moss_tts_nano.py b/tests/e2e/online_serving/test_moss_tts_nano.py new file mode 100644 index 000000000..662114db3 --- /dev/null +++ b/tests/e2e/online_serving/test_moss_tts_nano.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E Online tests for MOSS-TTS-Nano model with text input and audio output. + +These tests verify the /v1/audio/speech endpoint works correctly with +actual model inference, not mocks. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + +from pathlib import Path + +import pytest + +from tests.conftest import OmniServerParams +from tests.utils import hardware_test + +MODEL = "OpenMOSS-Team/MOSS-TTS-Nano" + + +def get_stage_config(name: str = "moss_tts_nano.yaml"): + """Get the stage config path from vllm_omni model_executor stage_configs.""" + return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name) + + +def get_prompt(prompt_type="text"): + """Text prompt for text-to-audio tests.""" + prompts = { + "text": "Hello, this is a test of MOSS-TTS-Nano text to speech synthesis.", + "chinese": "你好，这是MOSS-TTS-Nano的语音合成测试。", + } + return prompts.get(prompt_type, prompts["text"]) + + +tts_server_params = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=get_stage_config("moss_tts_nano.yaml"), + server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"], + ), + id="moss_tts_nano", + ) +] + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_audio_001(omni_server, openai_client) -> None: + """ + Test text input processing and audio output via /v1/audio/speech. + Deploy Setting: default yaml + Input Modal: text + Output Modal: audio (48 kHz) + Input Setting: stream=False, voice=Junhao + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": False, + "response_format": "wav", + "voice": "Junhao", + } + + openai_client.send_audio_speech_request(request_config) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_audio_002(omni_server, openai_client) -> None: + """ + Test streaming text-to-audio via /v1/audio/speech. + Deploy Setting: default yaml + Input Modal: text + Output Modal: audio (48 kHz, PCM stream) + Input Setting: stream=True, voice=Ava + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt(), + "stream": True, + "response_format": "pcm", + "voice": "Ava", + } + + openai_client.send_audio_speech_request(request_config) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +def test_text_to_audio_003(omni_server, openai_client) -> None: + """ + Test Chinese text-to-audio via /v1/audio/speech. + Deploy Setting: default yaml + Input Modal: text (Chinese) + Output Modal: audio (48 kHz) + Input Setting: stream=False, voice=Junhao + Datasets: single request + """ + request_config = { + "model": omni_server.model, + "input": get_prompt("chinese"), + "stream": False, + "response_format": "wav", + "voice": "Junhao", + } + + openai_client.send_audio_speech_request(request_config)