diff --git a/examples/offline_inference/moss_tts_nano/README.md b/examples/offline_inference/moss_tts_nano/README.md new file mode 100644 index 000000000..f7ace3b86 --- /dev/null +++ b/examples/offline_inference/moss_tts_nano/README.md @@ -0,0 +1,99 @@ +# MOSS-TTS-Nano Offline Inference + +## Overview + +Single-stage offline TTS pipeline using the 0.1B MOSS-TTS-Nano AR LM and MOSS-Audio-Tokenizer-Nano codec. Outputs 48 kHz stereo WAV. + +## Quick Start + +```bash +python end2end.py --text "Hello, this is MOSS-TTS-Nano." +``` + +The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face. + +## Usage + +``` +python end2end.py [OPTIONS] + +Options: + --text TEXT Text to synthesize (default: "Hello, this is MOSS-TTS-Nano speaking.") + --voice VOICE Built-in voice preset (default: Junhao). See voice table below. + --mode MODE voice_clone (default) or continuation + --prompt-audio PATH Reference WAV/MP3 for custom voice cloning + --prompt-text TEXT Reference transcript (continuation mode) + --max-new-frames N Max AR frames, default 375 (~14 s audio) + --seed INT Random seed for reproducibility + --audio-temperature F Audio sampling temperature (default: 0.8) + --audio-top-k N Audio top-k sampling (default: 25) + --audio-top-p F Audio top-p sampling (default: 0.95) + --text-temperature F Text layer temperature (default: 1.0) + --batch Run a built-in batch of diverse samples (ZH/EN/FR) + --output-dir DIR Directory for WAV outputs (default: /tmp/moss_tts_nano_output) + --stage-config PATH Path to stage config YAML + --stage-init-timeout INT Timeout in seconds for stage init (default: 120) +``` + +## Examples + +```bash +# Built-in Chinese voice +python end2end.py --text "你好，这是MOSS-TTS-Nano的语音合成演示。" --voice Junhao + +# Built-in English voice +python end2end.py --text "Hello from MOSS-TTS-Nano." --voice Ava + +# Custom voice clone +python end2end.py \ + --text "Hello, this is a cloned voice." \ + --prompt-audio /path/to/reference.wav \ + --prompt-text "Exact transcript of the reference audio." + +# Batch synthesis (ZH/EN/FR) +python end2end.py --batch --output-dir /tmp/batch_output + +# Reproducible output +python end2end.py --text "Deterministic test." --seed 42 +``` + +## Built-in Voice Presets + +| Voice | Language | +|-------|----------| +| `Junhao` | ZH | +| `Zhiming` | ZH | +| `Weiguo` | ZH | +| `Xiaoyu` | ZH | +| `Yuewen` | ZH | +| `Lingyu` | ZH | +| `Ava` | EN | +| `Bella` | EN | +| `Adam` | EN | +| `Nathan` | EN | +| `Sakura` | JA | +| `Yui` | JA | +| `Aoi` | JA | +| `Hina` | JA | +| `Mei` | JA | + +## Stage Config + +The stage config is at `vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml`. Key settings: + +```yaml +engine_args: + gpu_memory_utilization: 0.3 # ~2 GB VRAM; increase for faster init + max_num_seqs: 4 # concurrent requests + max_model_len: 4096 +``` + +## Output Format + +WAV files, 48 kHz, stereo (2-channel). The codec interleaves stereo as `[L, R, L, R, ...]` in the flat tensor returned by the model. + +## Troubleshooting + +- **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ torchcodec backend requires NVRTC. The model patches `torchaudio.load/save` automatically at load time to fall back to soundfile. +- **`flash_attn not installed`**: The model falls back to `sdpa` attention automatically. +- **Empty audio**: Check that `--text` is non-empty and the model loaded successfully (look for "MOSS-TTS-Nano LM loaded" in logs). diff --git a/examples/online_serving/moss_tts_nano/README.md b/examples/online_serving/moss_tts_nano/README.md new file mode 100644 index 000000000..371bde789 --- /dev/null +++ b/examples/online_serving/moss_tts_nano/README.md @@ -0,0 +1,159 @@ +# MOSS-TTS-Nano + +## Model + +| Model | Description | +|-------|-------------| +| `OpenMOSS-Team/MOSS-TTS-Nano` | 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec, 48 kHz stereo, ZH/EN/JA | + +## Gradio Demo + +An interactive Gradio demo is available with multilingual voice presets, custom voice cloning, and streaming support. + +```bash +# Option 1: Launch server + Gradio together +./run_gradio_demo.sh + +# Option 2: If server is already running +python gradio_demo.py --api-base http://localhost:8091 +``` + +Then open http://localhost:7860 in your browser. + +Features: + +- 15 built-in voice presets (6 ZH · 4 EN · 5 JA) +- Custom voice cloning from uploaded reference audio +- Streaming mode (progressive PCM playback) + +## Launch the Server + +```bash +vllm-omni serve OpenMOSS-Team/MOSS-TTS-Nano \ + --stage-configs-path vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager +``` + +Or use the convenience script: + +```bash +./run_server.sh +``` + +## Send TTS Request + +### Using curl + +```bash +# Built-in voice preset +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "你好，这是语音合成测试。", + "voice": "Junhao", + "response_format": "wav" + }' --output output.wav + +# English preset +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is MOSS-TTS-Nano.", + "voice": "Ava", + "response_format": "wav" + }' --output output_en.wav +``` + +### Custom Voice Cloning + +Provide a reference audio (base64 data URL) and its transcript: + +```bash +REF_AUDIO=$(base64 -w 0 /path/to/reference.wav) +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d "{ + \"input\": \"Hello, this is a cloned voice.\", + \"voice\": \"Junhao\", + \"ref_audio\": \"data:audio/wav;base64,${REF_AUDIO}\", + \"ref_text\": \"Exact transcript of the reference audio.\" + }" --output cloned.wav +``` + +### Using Python + +```python +import httpx + +response = httpx.post( + "http://localhost:8091/v1/audio/speech", + json={ + "input": "你好，这是语音合成测试。", + "voice": "Junhao", + "response_format": "wav", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +### Streaming + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, streaming output from MOSS-TTS-Nano.", + "voice": "Ava", + "stream": true, + "response_format": "pcm" + }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 2 - +``` + +**Note:** MOSS-TTS-Nano outputs at 48 kHz stereo (2-channel). + +## API Parameters + +MOSS-TTS-Nano uses the standard `/v1/audio/speech` endpoint. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | **required** | Text to synthesize (ZH / EN / JA) | +| `voice` | string | `"Junhao"` | Built-in voice name (see table below) | +| `response_format` | string | `"wav"` | Audio format: wav, mp3, flac, pcm | +| `ref_audio` | string | null | Base64 data URL for custom voice cloning | +| `ref_text` | string | null | Transcript of reference audio (required when `ref_audio` is set) | +| `stream` | bool | false | Stream raw PCM chunks | +| `max_new_tokens` | int | 4096 | Maximum tokens to generate | + +## Built-in Voice Presets + +| Voice | Language | Description | +|-------|----------|-------------| +| `Junhao` | ZH | Male, standard Mandarin | +| `Zhiming` | ZH | Male | +| `Weiguo` | ZH | Male | +| `Xiaoyu` | ZH | Female | +| `Yuewen` | ZH | Female | +| `Lingyu` | ZH | Female | +| `Ava` | EN | Female, American English | +| `Bella` | EN | Female | +| `Adam` | EN | Male | +| `Nathan` | EN | Male | +| `Sakura` | JA | Female | +| `Yui` | JA | Female | +| `Aoi` | JA | Female | +| `Hina` | JA | Female | +| `Mei` | JA | Female | + +## Troubleshooting + +1. **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ defaults to torchcodec which requires NVRTC. vLLM-Omni patches this automatically at model load time to use soundfile instead. +2. **Connection refused**: Ensure the server is running on the correct port. +3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings. +4. **Out of memory**: The default `gpu_memory_utilization=0.3` is conservative. Increase it in the stage config if you have more VRAM available. diff --git a/examples/online_serving/moss_tts_nano/gradio_demo.py b/examples/online_serving/moss_tts_nano/gradio_demo.py new file mode 100644 index 000000000..6eeb8d4c0 --- /dev/null +++ b/examples/online_serving/moss_tts_nano/gradio_demo.py @@ -0,0 +1,693 @@ +"""Gradio demo for MOSS-TTS-Nano with gapless streaming audio playback. + +Uses a custom AudioWorklet-based player (adapted from the Qwen3-TTS demo) +for gap-free streaming. Audio is streamed from the vLLM-Omni server +through a same-origin proxy and played via the Web Audio API's AudioWorklet. + +MOSS-TTS-Nano outputs 48 kHz stereo (2-channel interleaved L,R,L,R,...). +The AudioWorklet downmixes stereo to mono during playback. + +Also supports non-streaming mode (full audio download) via gr.Audio. + +Usage: + # Start the server first (see run_server.sh), then: + python gradio_demo.py --api-base http://localhost:8091 + + # Or use run_gradio_demo.sh to start both server and demo together. +""" + +from __future__ import annotations + +import argparse +import base64 +import io +import json +import logging + +try: + import gradio as gr +except ImportError: + raise ImportError( + "gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'" + ) from None + +import httpx +import numpy as np +import soundfile as sf +from fastapi import FastAPI, Request +from fastapi.responses import Response, StreamingResponse + +logger = logging.getLogger(__name__) + +# MOSS-TTS-Nano outputs 48 kHz stereo (2-channel interleaved). +PCM_SAMPLE_RATE = 48000 +PCM_CHANNELS = 2 +DEFAULT_API_BASE = "http://localhost:8091" + +# Built-in voice presets (matches the MOSS-TTS-Nano GitHub repo assets/) +VOICE_PRESETS = { + # Chinese + "Junhao (ZH)": "Junhao", + "Zhiming (ZH)": "Zhiming", + "Weiguo (ZH)": "Weiguo", + "Xiaoyu (ZH)": "Xiaoyu", + "Yuewen (ZH)": "Yuewen", + "Lingyu (ZH)": "Lingyu", + # English + "Ava (EN)": "Ava", + "Bella (EN)": "Bella", + "Adam (EN)": "Adam", + "Nathan (EN)": "Nathan", + # Japanese + "Sakura (JA)": "Sakura", + "Yui (JA)": "Yui", + "Aoi (JA)": "Aoi", + "Hina (JA)": "Hina", + "Mei (JA)": "Mei", +} +PRESET_LABELS = list(VOICE_PRESETS.keys()) +DEFAULT_PRESET = "Junhao (ZH)" + +# ── AudioWorklet processor (loaded in browser via Blob URL) ────────── +# Downmixes stereo-interleaved int16 PCM to mono float32 for playback. +WORKLET_JS = r""" +class TTSPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.buf = null; + this.pos = 0; + this.playing = false; + this.played = 0; + this.port.onmessage = (e) => { + if (e.data && e.data.type === 'clear') { + this.queue = []; this.buf = null; this.pos = 0; this.played = 0; + if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped'}); } + return; + } + // Input: Int16Array of stereo-interleaved samples [L0,R0,L1,R1,...] + // Convert to mono float32 by averaging L+R channels. + const pcm = e.data; + const monoLen = Math.floor(pcm.length / 2); + const mono = new Float32Array(monoLen); + for (let i = 0; i < monoLen; i++) { + mono[i] = (pcm[i * 2] + pcm[i * 2 + 1]) / 65536.0; + } + this.queue.push(mono); + }; + } + process(inputs, outputs) { + const out = outputs[0][0]; + for (let i = 0; i < out.length; i++) { + if (!this.buf || this.pos >= this.buf.length) { + if (this.queue.length > 0) { + this.buf = this.queue.shift(); this.pos = 0; + } else { + for (let j = i; j < out.length; j++) out[j] = 0; + if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped', played:this.played}); } + return true; + } + } + out[i] = this.buf[this.pos++]; + this.played++; + } + if (!this.playing) { this.playing = true; this.port.postMessage({type:'started'}); } + return true; + } +} +registerProcessor('tts-playback-processor', TTSPlaybackProcessor); +""" + +# ── Player HTML (container with metric cards) ──────────────────────── +PLAYER_HTML = """ +
+
+
+ Ready + +
+ + + +
+""" + + +def _build_player_js(sample_rate: int) -> str: + """Build the JavaScript that powers the AudioWorklet player.""" + return f""" + +""" + + +def encode_audio_to_base64(audio_data: tuple) -> str: + """Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL.""" + sample_rate, audio_np = audio_data + if audio_np.dtype != np.int16: + if audio_np.dtype in (np.float32, np.float64): + audio_np = np.clip(audio_np, -1.0, 1.0) + audio_np = (audio_np * 32767).astype(np.int16) + else: + audio_np = audio_np.astype(np.int16) + buf = io.BytesIO() + sf.write(buf, audio_np, sample_rate, format="WAV") + wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8") + return f"data:audio/wav;base64,{wav_b64}" + + +def build_payload( + text: str, + voice_preset: str, + ref_audio: tuple | None, + ref_text: str, + response_format: str = "pcm", + stream: bool = True, +) -> dict: + """Build the /v1/audio/speech request payload.""" + if not text or not text.strip(): + raise gr.Error("Please enter text to synthesize.") + + voice_name = VOICE_PRESETS.get(voice_preset, "Junhao") + + payload: dict = { + "input": text.strip(), + "voice": voice_name, + "response_format": "pcm" if stream else response_format, + "stream": stream, + } + + has_ref = ref_audio is not None + if has_ref: + payload["ref_audio"] = encode_audio_to_base64(ref_audio) + ref_text_stripped = ref_text.strip() if ref_text else "" + if ref_text_stripped: + payload["ref_text"] = ref_text_stripped + else: + raise gr.Error("Voice cloning requires a transcript of the reference audio.") + + return payload + + +def generate_speech( + api_base: str, + text: str, + voice_preset: str, + ref_audio: tuple | None, + ref_text: str, + response_format: str, +) -> tuple: + """Non-streaming: call /v1/audio/speech and return full audio as (sr, np_array).""" + payload = build_payload(text, voice_preset, ref_audio, ref_text, response_format, stream=False) + + try: + with httpx.Client(timeout=300.0) as client: + resp = client.post( + f"{api_base}/v1/audio/speech", + json=payload, + headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"}, + ) + except httpx.TimeoutException: + raise gr.Error("Request timed out. The server may be busy.") + except httpx.ConnectError: + raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") + + if resp.status_code != 200: + raise gr.Error(f"Server error ({resp.status_code}): {resp.text}") + + try: + if response_format == "pcm": + # PCM is stereo interleaved; downmix to mono for Gradio. + samples = np.frombuffer(resp.content, dtype=np.int16) + n = len(samples) - (len(samples) % 2) + left = samples[:n:2].astype(np.float32) + right = samples[1:n:2].astype(np.float32) + mono = (left + right) / 65536.0 + return (PCM_SAMPLE_RATE, mono) + # WAV/MP3/FLAC: soundfile handles header parsing. + audio_np, sample_rate = sf.read(io.BytesIO(resp.content)) + if audio_np.ndim > 1: + audio_np = audio_np.mean(axis=1) + return (int(sample_rate), audio_np.astype(np.float32)) + except Exception as e: + raise gr.Error(f"Failed to decode audio: {e}") + + +def create_app(api_base: str): + """Create the FastAPI app with streaming proxy + Gradio UI.""" + fastapi_app = FastAPI() + + # ── Streaming proxy (same-origin, no CORS issues) ──────────── + @fastapi_app.post("/proxy/v1/audio/speech") + async def proxy_speech(request: Request): + body = await request.json() + logger.info( + "Proxy request: input=%r voice=%s stream=%s", + body.get("input", "")[:50], + body.get("voice"), + body.get("stream"), + ) + try: + client = httpx.AsyncClient(timeout=300) + resp = await client.send( + client.build_request( + "POST", + f"{api_base}/v1/audio/speech", + json=body, + headers={"Authorization": "Bearer EMPTY", "Content-Type": "application/json"}, + ), + stream=True, + ) + except Exception as exc: + logger.exception("Proxy connection error") + await client.aclose() + return Response(content=str(exc), status_code=502) + + if resp.status_code != 200: + content = await resp.aread() + logger.error("Proxy upstream error %d: %s", resp.status_code, content[:200]) + await resp.aclose() + await client.aclose() + return Response(content=content, status_code=resp.status_code) + + async def relay(): + total = 0 + try: + async for chunk in resp.aiter_bytes(): + total += len(chunk) + yield chunk + except Exception: + logger.exception("Proxy relay error after %d bytes", total) + finally: + logger.info("Proxy relay done: %d bytes", total) + await resp.aclose() + await client.aclose() + + return StreamingResponse(relay(), media_type="application/octet-stream") + + # ── Gradio UI ──────────────────────────────────────────────── + css = """ + #generate-btn button { width: 100%; } + #streaming-player { border: 1px solid var(--border-color-primary) !important; border-radius: var(--block-radius) !important; padding: var(--block-padding) !important; } + """ + + theme = gr.themes.Default( + primary_hue=gr.themes.Color( + c50="#E8F5E9", + c100="#C8E6C9", + c200="#A5D6A7", + c300="#81C784", + c400="#66BB6A", + c500="#4CAF50", + c600="#43A047", + c700="#388E3C", + c800="#2E7D32", + c900="#1B5E20", + c950="#0D3B10", + ), + ) + + with gr.Blocks(title="MOSS-TTS-Nano Demo") as demo: + gr.HTML(f""" +
+ vLLM-Omni +
+

MOSS-TTS-Nano Streaming Demo

+ + 0.1B AR LM + MOSS-Audio-Tokenizer-Nano · 48 kHz · ZH / EN / JA +  ·  Served by + vLLM-Omni +  ·  {api_base} + +
+
+ """) + + with gr.Row(): + with gr.Column(scale=3): + text_input = gr.Textbox( + label="Text to Synthesize", + placeholder="Enter text in Chinese, English, or Japanese...", + lines=4, + ) + + voice_preset = gr.Dropdown( + choices=PRESET_LABELS, + value=DEFAULT_PRESET, + label="Voice Preset", + info="Built-in multilingual voices. ZH = Chinese, EN = English, JA = Japanese.", + ) + + with gr.Accordion("Custom Voice Clone (optional)", open=False): + gr.Markdown( + "Upload a reference audio clip (10-30 s) and provide its transcript " + "to clone that voice instead of using the built-in preset." + ) + ref_audio = gr.Audio( + label="Reference Audio", + type="numpy", + sources=["upload", "microphone"], + ) + ref_text = gr.Textbox( + label="Reference Audio Transcript (required for cloning)", + placeholder="Exact transcript of the reference audio...", + lines=2, + ) + + with gr.Row(): + response_format = gr.Dropdown( + choices=["wav", "mp3", "flac", "pcm"], + value="pcm", + label="Audio Format", + interactive=False, + scale=1, + ) + stream_checkbox = gr.Checkbox( + label="Stream (gapless)", + value=True, + info="AudioWorklet streaming (recommended)", + scale=1, + ) + + with gr.Row(): + generate_btn = gr.Button( + "Generate Speech", + variant="primary", + size="lg", + elem_id="generate-btn", + scale=3, + ) + reset_btn = gr.Button( + "Reset", + variant="secondary", + size="lg", + scale=1, + ) + + with gr.Column(scale=2): + player_html = gr.HTML( + value=PLAYER_HTML, + visible=True, + label="streaming player", + elem_id="streaming-player", + ) + audio_output = gr.Audio( + label="Generated Audio", + interactive=False, + autoplay=True, + visible=False, + ) + gr.Examples( + examples=[ + ["Hello, this is MOSS-TTS-Nano speaking.", "Ava (EN)"], + ["The quick brown fox jumps over the lazy dog.", "Adam (EN)"], + ["MOSS-TTS-Nanoは、軽量なテキスト読み上げモデルです。", "Sakura (JA)"], + ], + inputs=[text_input, voice_preset], + label="Examples", + ) + gr.HTML(""" +
+ + vLLM-Omni + +
+ """) + + # Hidden textbox to pass payload from Python -> JavaScript + hidden_payload = gr.Textbox(visible=False, elem_id="tts-payload") + + # ── Event wiring ───────────────────────────────────────── + def on_stream_change(stream: bool): + if stream: + return ( + gr.update(value="pcm", interactive=False), + gr.update(visible=True), # player + gr.update(visible=False), # audio + ) + return ( + gr.update(value="wav", interactive=True), + gr.update(visible=False), + gr.update(visible=True), + ) + + stream_checkbox.change( + fn=on_stream_change, + inputs=[stream_checkbox], + outputs=[response_format, player_html, audio_output], + ) + + def on_reset(): + return ( + "", # text + None, # audio_output + "", # hidden_payload + PLAYER_HTML, # reset player + ) + + reset_btn.click( + fn=on_reset, + outputs=[text_input, audio_output, hidden_payload, player_html], + js="() => { if (window.ttsStop) window.ttsStop(); }", + ) + + all_inputs = [text_input, voice_preset, ref_audio, ref_text, response_format] + + def on_generate(stream_enabled, *args): + import time as _time + + text, voice, ref_a, ref_t, _fmt = args + if stream_enabled: + payload = build_payload(text, voice, ref_a, ref_t, stream=True) + payload["_nonce"] = int(_time.time() * 1000) + return json.dumps(payload), gr.update() + else: + audio = generate_speech(api_base, text, voice, ref_a, ref_t, _fmt) + return "", audio + + generate_btn.click( + fn=on_generate, + inputs=[stream_checkbox] + all_inputs, + outputs=[hidden_payload, audio_output], + ).then( + fn=lambda p: p, + inputs=[hidden_payload], + outputs=[hidden_payload], + js="(p) => { if (p && p.trim()) { const d = JSON.parse(p); delete d._nonce; window.ttsGenerate(d); } return p; }", + ) + + demo.queue() + + return gr.mount_gradio_app( + fastapi_app, + demo, + path="/", + css=css, + theme=theme, + head=_build_player_js(PCM_SAMPLE_RATE), + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Gradio demo for MOSS-TTS-Nano with gapless AudioWorklet streaming.", + ) + parser.add_argument( + "--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})" + ) + parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)") + parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)") + parser.add_argument("--share", action="store_true", help="Share publicly via Gradio tunnel") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + print(f"Connecting to vLLM-Omni server at: {args.api_base}") + + app = create_app(args.api_base) + + import uvicorn + uvicorn.run(app, host=args.host, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/moss_tts_nano/run_gradio_demo.sh b/examples/online_serving/moss_tts_nano/run_gradio_demo.sh new file mode 100755 index 000000000..1fb5d1914 --- /dev/null +++ b/examples/online_serving/moss_tts_nano/run_gradio_demo.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Launch MOSS-TTS-Nano server + Gradio demo together. +# +# Usage: +# ./run_gradio_demo.sh +# CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh + +set -e + +MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}" +PORT="${PORT:-8091}" +GRADIO_PORT="${GRADIO_PORT:-7860}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +echo "Starting MOSS-TTS-Nano server (port $PORT)..." +FLASHINFER_DISABLE_VERSION_CHECK=1 \ +vllm-omni serve "$MODEL" \ + --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --enforce-eager \ + --omni & +SERVER_PID=$! + +cleanup() { + echo "Stopping server (PID $SERVER_PID)..." + kill $SERVER_PID 2>/dev/null + wait $SERVER_PID 2>/dev/null +} +trap cleanup EXIT + +# Wait for server to be ready. +echo "Waiting for server to start..." +for i in $(seq 1 120); do + if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then + echo "Server ready." + break + fi + sleep 2 +done + +echo "Starting Gradio demo (port $GRADIO_PORT)..." +python "$SCRIPT_DIR/gradio_demo.py" \ + --api-base "http://localhost:$PORT" \ + --port "$GRADIO_PORT" diff --git a/examples/online_serving/moss_tts_nano/run_server.sh b/examples/online_serving/moss_tts_nano/run_server.sh new file mode 100755 index 000000000..c5254c428 --- /dev/null +++ b/examples/online_serving/moss_tts_nano/run_server.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Launch vLLM-Omni server for MOSS-TTS-Nano +# +# Usage: +# ./run_server.sh +# CUDA_VISIBLE_DEVICES=0 PORT=8091 ./run_server.sh + +set -e + +MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}" +PORT="${PORT:-8091}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +echo "Starting MOSS-TTS-Nano server with model: $MODEL" + +FLASHINFER_DISABLE_VERSION_CHECK=1 \ +vllm-omni serve "$MODEL" \ + --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --enforce-eager \ + --omni diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 1d9754853..d4d57ae0f 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -50,6 +50,7 @@ _FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"} _COSYVOICE3_TTS_MODEL_STAGES = {"cosyvoice3_talker"} _OMNIVOICE_TTS_MODEL_STAGES = {"omnivoice_generator"} _VOXCPM2_TTS_MODEL_STAGES = {"latent_generator"} +_MOSS_TTS_MODEL_STAGES = {"moss_tts_nano"} _TTS_MODEL_STAGES: set[str] = ( _VOXTRAL_TTS_MODEL_STAGES | _QWEN3_TTS_MODEL_STAGES @@ -57,6 +58,7 @@ _TTS_MODEL_STAGES: set[str] = ( | _COSYVOICE3_TTS_MODEL_STAGES | _OMNIVOICE_TTS_MODEL_STAGES | _VOXCPM2_TTS_MODEL_STAGES + | _MOSS_TTS_MODEL_STAGES ) _TTS_LANGUAGES: set[str] = { "Auto", @@ -294,6 +296,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return "omnivoice" if model_stage in _VOXCPM2_TTS_MODEL_STAGES: return "voxcpm2" + if model_stage in _MOSS_TTS_MODEL_STAGES: + return "moss_tts_nano" return None def _compute_max_instructions_length(self) -> int: @@ -793,6 +797,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return self._validate_cosyvoice3_request(request) if self._tts_model_type == "voxcpm2": return None # VoxCPM2 accepts any text input + if self._tts_model_type == "moss_tts_nano": + return self._validate_moss_tts_request(request) return self._validate_qwen_tts_request(request) def _validate_ref_audio_format(self, ref_audio: str) -> str | None: @@ -957,6 +963,30 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return None + def _validate_moss_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: + """Validate MOSS-TTS-Nano request. Only requires non-empty input text.""" + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + return None + + def _build_moss_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: + """Build additional_information for MOSS-TTS-Nano. + + Maps the standard /v1/audio/speech fields to MOSS-TTS-Nano's + additional_information keys (text, voice, mode, prompt_audio_path, etc.). + Values are wrapped in lists to match the vllm-omni convention. + """ + params: dict[str, Any] = { + "text": [request.input], + } + if request.voice is not None: + params["voice"] = [request.voice] + if request.ref_text is not None: + params["prompt_text"] = [request.ref_text] + if request.max_new_tokens is not None: + params["max_new_frames"] = [request.max_new_tokens] + return params + def _validate_fish_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate Fish Speech request parameters. Returns error message or None. @@ -1456,6 +1486,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): elif self._tts_model_type == "cosyvoice3": prompt = await self._build_cosyvoice3_prompt(request) tts_params = {} + elif self._tts_model_type == "moss_tts_nano": + tts_params = self._build_moss_tts_params(request) + prompt = {"prompt_token_ids": [1], "additional_information": tts_params} else: tts_params = self._build_tts_params(request) # Resolve ref_audio (explicit or auto-set for uploaded voices) diff --git a/vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py b/vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py index e2be3ea7e..61e320571 100644 --- a/vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py +++ b/vllm_omni/model_executor/models/moss_tts_nano/modeling_moss_tts_nano.py @@ -93,7 +93,7 @@ _DEFAULT_AUDIO_TOP_K = 25 _DEFAULT_AUDIO_REPETITION_PENALTY = 1.2 _DEFAULT_MAX_NEW_FRAMES = 375 _DEFAULT_VOICE = "Junhao" -_DEFAULT_MODE = "voice_clone" +_DEFAULT_MODE = "continuation" def _pick(info: dict, key: str, default): @@ -149,9 +149,6 @@ class MossTTSNanoForGeneration(nn.Module): self._device: torch.device | None = None self._lock = threading.Lock() - # Dummy parameter so vLLM can resolve the device via next(parameters()). - self._sentinel = nn.Parameter(torch.zeros(1), requires_grad=False) - # ------------------------------------------------------------------ # Weight loading # ------------------------------------------------------------------ @@ -405,6 +402,13 @@ class MossTTSNanoForGeneration(nn.Module): # vLLM boilerplate: embed_input_ids required by generation runner # ------------------------------------------------------------------ + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + # Generation stage does not produce logits; satisfy VllmModelForTextGeneration protocol. + return None + def embed_input_ids( self, input_ids: torch.Tensor, diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index f8f11cb81..2411a388c 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -166,7 +166,7 @@ _OMNI_MODELS = { ## MOSS-TTS-Nano "MossTTSNanoForCausalLM": ( "moss_tts_nano", - "moss_tts_nano", + "modeling_moss_tts_nano", "MossTTSNanoForGeneration", ), "DyninOmniForConditionalGeneration": ( diff --git a/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml b/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml index 1155247e9..7e27f05cf 100644 --- a/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml +++ b/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml @@ -26,7 +26,7 @@ async_chunk: false stage_args: - stage_id: 0 stage_type: llm - is_comprehension: false + is_comprehension: true runtime: devices: "0" engine_args: