From 8b102bdac1dfd04cf09e3dd9e3b6068a506a2697 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Tue, 24 Mar 2026 00:06:41 -0400 Subject: [PATCH] [Feat][Qwen3-TTS] Better Qwen3-TTS online serving demo (#1857) Signed-off-by: linyueqian --- .../online_serving/qwen3_tts/gradio_demo.py | 797 +++++++++++++----- .../qwen3_tts/gradio_fastrtc_demo.py | 274 ------ .../qwen3_tts/run_gradio_demo.sh | 3 +- .../online_serving/qwen3_tts/run_server.sh | 1 - .../online_serving/qwen3_tts/tts_common.py | 4 +- 5 files changed, 605 insertions(+), 474 deletions(-) delete mode 100644 examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py diff --git a/examples/online_serving/qwen3_tts/gradio_demo.py b/examples/online_serving/qwen3_tts/gradio_demo.py index b8a54c6a6..a2f17cb78 100644 --- a/examples/online_serving/qwen3_tts/gradio_demo.py +++ b/examples/online_serving/qwen3_tts/gradio_demo.py @@ -1,9 +1,13 @@ -"""Gradio demo for Qwen3-TTS online serving via /v1/audio/speech API. +"""Gradio demo for Qwen3-TTS with gapless streaming audio playback. -Uses Gradio's built-in audio component for playback. Supports streaming -(progressive PCM chunks) and non-streaming (full audio download) modes. +Uses a custom AudioWorklet-based player for gap-free streaming, +inspired by github.com/KoljaB/RealtimeVoiceChat. Audio is streamed +from the vLLM server through a same-origin proxy and played via the +Web Audio API's AudioWorklet, which maintains a FIFO buffer queue +and plays samples at the audio clock rate — eliminating inter-chunk +gaps inherent in Gradio's built-in streaming audio component. -For gapless streaming playback, see gradio_fastrtc_demo.py which uses WebRTC. +Also supports non-streaming mode (full audio download) via gr.Audio. Supports all 3 task types: - CustomVoice: Predefined speaker with optional style instructions @@ -11,19 +15,21 @@ Supports all 3 task types: - Base: Voice cloning from reference audio (upload or URL) Usage: - # Start the server first (see run_server.sh), then: + # Start the vLLM server first (see run_server.sh), then: python gradio_demo.py --api-base http://localhost:8000 - - # Or use run_gradio_demo.sh to start both server and demo together. """ import argparse import io +import json +import logging import gradio as gr import httpx import numpy as np import soundfile as sf +from fastapi import FastAPI, Request +from fastapi.responses import Response, StreamingResponse from tts_common import ( PCM_SAMPLE_RATE, SUPPORTED_LANGUAGES, @@ -31,10 +37,252 @@ from tts_common import ( add_common_args, build_payload, fetch_voices, - on_task_type_change, - stream_pcm_chunks, ) +logger = logging.getLogger(__name__) + +# ── AudioWorklet processor (loaded in browser via Blob URL) ────────── +WORKLET_JS = r""" +class TTSPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.buf = null; + this.pos = 0; + this.playing = false; + this.played = 0; + this.port.onmessage = (e) => { + if (e.data && e.data.type === 'clear') { + this.queue = []; this.buf = null; this.pos = 0; this.played = 0; + if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped'}); } + return; + } + this.queue.push(e.data); + }; + } + process(inputs, outputs) { + const out = outputs[0][0]; + for (let i = 0; i < out.length; i++) { + if (!this.buf || this.pos >= this.buf.length) { + if (this.queue.length > 0) { + this.buf = this.queue.shift(); this.pos = 0; + } else { + for (let j = i; j < out.length; j++) out[j] = 0; + if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped', played:this.played}); } + return true; + } + } + out[i] = this.buf[this.pos++] / 32768; + this.played++; + } + if (!this.playing) { this.playing = true; this.port.postMessage({type:'started'}); } + return true; + } +} +registerProcessor('tts-playback-processor', TTSPlaybackProcessor); +""" + +# ── Player HTML (container with metric cards) ──────────────────────── +PLAYER_HTML = """ +
+
+
+ Ready + +
+ + + +
+""" + + +def _build_player_js(sample_rate: int) -> str: + """Build the JavaScript that powers the AudioWorklet player.""" + return f""" +() => {{ + const SR = {sample_rate}; + const WC = {json.dumps(WORKLET_JS)}; + let ctx = null, node = null, abort = null, gen = false, st = {{}}; + + async function init() {{ + if (ctx) return; + ctx = new AudioContext({{ sampleRate: SR }}); + const b = new Blob([WC], {{ type: 'application/javascript' }}); + const u = URL.createObjectURL(b); + await ctx.audioWorklet.addModule(u); + URL.revokeObjectURL(u); + node = new AudioWorkletNode(ctx, 'tts-playback-processor'); + node.connect(ctx.destination); + node.port.onmessage = (e) => {{ + if (e.data.type === 'started') setStatus('Playing...', '#64dd17'); + else if (e.data.type === 'stopped' && !gen) {{ + setStatus('Done', '#64dd17'); showStats(true); + const btn = document.getElementById('tts-stop-btn'); + if (btn) btn.style.display = 'none'; + }} + }}; + }} + + function setStatus(text, color) {{ + const s = document.getElementById('tts-status'); + const d = document.getElementById('tts-status-dot'); + if (s) s.textContent = text; + if (d) d.style.background = color || '#ccc'; + }} + + function showStats(fin) {{ + if (!st.t0) return; + const elapsed = (fin && st.streamEnd ? (st.streamEnd - st.t0) : (performance.now() - st.t0)) / 1000; + const dur = st.samples / SR; + const mTtfp = document.getElementById('tts-m-ttfp'); + const mRtf = document.getElementById('tts-m-rtf'); + const mDur = document.getElementById('tts-m-dur'); + const mSpeed = document.getElementById('tts-m-speed'); + const bar = document.getElementById('tts-rtf-bar'); + const barLabel = document.getElementById('tts-rtf-label'); + const elapsedEl = document.getElementById('tts-elapsed'); + + if (mTtfp && st.ttfp != null) mTtfp.textContent = st.ttfp.toFixed(0) + 'ms'; + if (mDur) mDur.textContent = dur.toFixed(1) + 's'; + + if (dur > 0 && elapsed > 0) {{ + const rtf = elapsed / dur; + const speed = 1 / rtf; + if (mRtf) {{ + mRtf.textContent = rtf.toFixed(2) + 'x'; + mRtf.style.color = rtf < 1 ? '#64dd17' : rtf < 1.5 ? '#e8a317' : '#EF5552'; + }} + if (mSpeed) {{ + mSpeed.textContent = speed.toFixed(1) + 'x'; + mSpeed.style.color = speed > 1 ? '#64dd17' : speed > 0.7 ? '#e8a317' : '#EF5552'; + }} + if (bar) {{ + const pct = Math.min(speed / 3 * 100, 100); + bar.style.width = pct + '%'; + bar.style.background = speed > 1 ? 'linear-gradient(90deg,#4A90D9,#64dd17)' : speed > 0.7 ? 'linear-gradient(90deg,#e8a317,#f0b866)' : 'linear-gradient(90deg,#EF5552,#f87171)'; + }} + if (barLabel) barLabel.textContent = speed.toFixed(1) + 'x realtime'; + }} + if (elapsedEl) {{ + elapsedEl.style.display = 'block'; + elapsedEl.textContent = fin ? 'Completed in ' + elapsed.toFixed(1) + 's (' + st.chunks + ' chunks)' : elapsed.toFixed(1) + 's elapsed (' + st.chunks + ' chunks)'; + }} + }} + + window.ttsStop = function() {{ + if (abort) abort.abort(); + if (node) node.port.postMessage({{ type: 'clear' }}); + gen = false; + setStatus('Stopped', '#999'); + const btn = document.getElementById('tts-stop-btn'); + if (btn) btn.style.display = 'none'; + }}; + + window.ttsGenerate = async function(payload) {{ + try {{ await init(); if (ctx.state === 'suspended') await ctx.resume(); }} + catch (e) {{ const s = document.getElementById('tts-status'); if (s) s.textContent = 'Audio init error: ' + e.message; return; }} + + // Abort previous request and clear worklet buffer + if (abort) abort.abort(); + node.port.postMessage({{ type: 'clear' }}); + // Wait for worklet to process clear before sending new data + await new Promise(r => setTimeout(r, 50)); + node.port.postMessage({{ type: 'clear' }}); + + gen = true; + st = {{ t0: performance.now(), chunks: 0, samples: 0, ttfp: null }}; + setStatus('Connecting...', '#4A90D9'); + const bEl = document.getElementById('tts-stop-btn'); + if (bEl) bEl.style.display = 'inline-block'; + const mp = document.getElementById('tts-metrics'); + if (mp) {{ mp.style.display = 'grid'; ['tts-m-ttfp','tts-m-rtf','tts-m-dur','tts-m-speed'].forEach(id => {{ const e = document.getElementById(id); if(e) {{ e.textContent = '—'; e.style.color = '#333'; }} }}); }} + const bw = document.getElementById('tts-rtf-bar-wrap'); + if (bw) bw.style.display = 'block'; + const bar = document.getElementById('tts-rtf-bar'); + if (bar) bar.style.width = '0%'; + const bl = document.getElementById('tts-rtf-label'); + if (bl) bl.textContent = ''; + const ee = document.getElementById('tts-elapsed'); + if (ee) {{ ee.style.display = 'none'; ee.textContent = ''; }} + abort = new AbortController(); + + try {{ + console.log('fetch payload:', JSON.stringify({{input: payload.input?.slice(0,30), task: payload.task_type, has_ref: !!payload.ref_audio, stream: payload.stream}})); + const r = await fetch('/proxy/v1/audio/speech', {{ + method: 'POST', + headers: {{ 'Content-Type': 'application/json' }}, + body: JSON.stringify(payload), + signal: abort.signal, + }}); + if (!r.ok) {{ const t = await r.text(); throw new Error('Server ' + r.status + ': ' + t.slice(0, 200)); }} + setStatus('Streaming...', '#4A90D9'); + + const reader = r.body.getReader(); + let left = new Uint8Array(0); + while (true) {{ + const {{ done, value }} = await reader.read(); + if (done) break; + let raw; + if (left.length > 0) {{ + raw = new Uint8Array(left.length + value.length); + raw.set(left); raw.set(value, left.length); + }} else {{ raw = value; }} + const usable = raw.length - (raw.length % 2); + left = usable < raw.length ? raw.slice(usable) : new Uint8Array(0); + if (usable > 0) {{ + const ab = new ArrayBuffer(usable); + new Uint8Array(ab).set(raw.subarray(0, usable)); + const pcm = new Int16Array(ab); + node.port.postMessage(pcm); + st.chunks++; + st.samples += pcm.length; + if (st.ttfp == null) st.ttfp = performance.now() - st.t0; + showStats(false); + }} + }} + }} catch (e) {{ + if (e.name !== 'AbortError') {{ + setStatus('Error: ' + e.message, '#EF5552'); + console.error('TTS error:', e); + }} + }} finally {{ + // Freeze RTF at stream-end time (before playback finishes) + st.streamEnd = performance.now(); + showStats(true); + gen = false; + if (st.samples > 0) setStatus('Finishing playback...', '#64dd17'); + else {{ + setStatus('No audio received', '#999'); + if (bEl) bEl.style.display = 'none'; + }} + }} + }}; +}} +""" + def generate_speech( api_base: str, @@ -50,7 +298,7 @@ def generate_speech( response_format: str, speed: float, ): - """Non-streaming: call /v1/audio/speech and return audio.""" + """Non-streaming: call /v1/audio/speech and return complete audio.""" payload = build_payload( text, task_type, @@ -65,21 +313,17 @@ def generate_speech( speed, stream=False, ) - try: with httpx.Client(timeout=300.0) as client: resp = client.post( f"{api_base}/v1/audio/speech", json=payload, - headers={ - "Content-Type": "application/json", - "Authorization": "Bearer EMPTY", - }, + headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"}, ) except httpx.TimeoutException: raise gr.Error("Request timed out. The server may be busy.") except httpx.ConnectError: - raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.") + raise gr.Error(f"Cannot connect to server at {api_base}.") if resp.status_code != 200: raise gr.Error(f"Server error ({resp.status_code}): {resp.text}") @@ -87,8 +331,7 @@ def generate_speech( content_type = resp.headers.get("content-type", "") if "application/json" in content_type: try: - error_data = resp.json() - raise gr.Error(f"Server error: {error_data}") + raise gr.Error(f"Server error: {resp.json()}") except ValueError: pass @@ -101,113 +344,133 @@ def generate_speech( audio_np = audio_np[:, 0] return (sample_rate, audio_np.astype(np.float32)) except Exception as e: - raise gr.Error(f"Failed to decode audio response: {e}") + raise gr.Error(f"Failed to decode audio: {e}") -def generate_speech_stream( - api_base: str, - text: str, - task_type: str, - voice: str, - language: str, - instructions: str, - ref_audio: tuple | None, - ref_audio_url: str, - ref_text: str, - x_vector_only: bool, - response_format: str, - speed: float, - chunk_seconds: float = 0.25, -): - """Streaming: yield individual PCM chunks for progressive playback.""" - payload = build_payload( - text, - task_type, - voice, - language, - instructions, - ref_audio, - ref_audio_url, - ref_text, - x_vector_only, - response_format, - speed, - stream=True, - ) +def create_app(api_base: str): + """Create the FastAPI app with streaming proxy + Gradio UI.""" + fastapi_app = FastAPI() - min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds) - # Pre-buffer 2 chunks before first playback to avoid the gap - # between chunk 1 and chunk 2 during progressive streaming. - prebuffer_chunks = 2 - pending: list[np.ndarray] = [] - pending_len = 0 - chunks_yielded = 0 - prebuffer: list[np.ndarray] = [] + # Server-side payload store (avoids sending large base64 through Gradio textbox) + _pending_payloads: dict[str, dict] = {} - try: - for samples in stream_pcm_chunks(api_base, payload): - float_samples = samples.astype(np.float32) / 32767.0 - pending.append(float_samples) - pending_len += len(float_samples) - if pending_len >= min_chunk_samples: - audio_chunk = np.concatenate(pending) - pending.clear() - pending_len = 0 + # ── Streaming proxy (same-origin, no CORS issues) ──────────── + @fastapi_app.post("/proxy/v1/audio/speech") + async def proxy_speech(request: Request): + body = await request.json() + # Check if this is a request ID referencing a stored payload + req_id = body.get("_req_id") + if req_id and req_id in _pending_payloads: + body = _pending_payloads.pop(req_id) + # Pre-download ref_audio URL so TTFP only measures synthesis time + ref = body.get("ref_audio", "") + if isinstance(ref, str) and ref.startswith("http"): + try: + async with httpx.AsyncClient(timeout=30) as dl: + r = await dl.get(ref) + r.raise_for_status() + import base64 - if chunks_yielded < prebuffer_chunks: - prebuffer.append(audio_chunk) - chunks_yielded += 1 - if chunks_yielded == prebuffer_chunks: - yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer)) - prebuffer.clear() - else: - yield (PCM_SAMPLE_RATE, audio_chunk) - # Flush: prebuffer (if stream was shorter than 2 chunks) + remaining pending - remaining = prebuffer + pending - if remaining: - yield (PCM_SAMPLE_RATE, np.concatenate(remaining)) - except httpx.TimeoutException: - raise gr.Error("Request timed out. The server may be busy.") - except httpx.ConnectError: - raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.") - - -def on_stream_change(stream: bool): - """When streaming is enabled, lock format to PCM and disable speed.""" - if stream: - return ( - gr.update(value="pcm", interactive=False), - gr.update(interactive=False), + b64 = base64.b64encode(r.content).decode() + ct = r.headers.get("content-type", "audio/wav") + body["ref_audio"] = f"data:{ct};base64,{b64}" + logger.info("Pre-downloaded ref_audio: %d bytes", len(r.content)) + except Exception: + logger.exception("Failed to pre-download ref_audio, passing URL as-is") + logger.info( + "Proxy request: %s", + {k: (f"<{len(str(v))} chars>" if k == "ref_audio" else v) for k, v in body.items()}, ) - return ( - gr.update(value="wav", interactive=True), - gr.update(interactive=True), - ) + try: + client = httpx.AsyncClient(timeout=300) + resp = await client.send( + client.build_request( + "POST", + f"{api_base}/v1/audio/speech", + json=body, + headers={"Authorization": "Bearer EMPTY", "Content-Type": "application/json"}, + ), + stream=True, + ) + except Exception as exc: + logger.exception("Proxy connection error") + await client.aclose() + return Response(content=str(exc), status_code=502) + if resp.status_code != 200: + content = await resp.aread() + logger.error("Proxy upstream error %d: %s", resp.status_code, content[:200]) + await resp.aclose() + await client.aclose() + return Response(content=content, status_code=resp.status_code) -def build_interface(api_base: str, stream_chunk_seconds: float = 0.25): - """Build the Gradio interface.""" + async def relay(): + total = 0 + try: + async for chunk in resp.aiter_bytes(): + total += len(chunk) + yield chunk + except Exception: + logger.exception("Proxy relay error after %d bytes", total) + finally: + logger.info("Proxy relay done: %d bytes", total) + await resp.aclose() + await client.aclose() + + return StreamingResponse(relay(), media_type="application/octet-stream") + + # ── Gradio UI ──────────────────────────────────────────────── voices = fetch_voices(api_base) css = """ #generate-btn button { width: 100%; } - .task-info { padding: 8px 12px; border-radius: 6px; - background: #f0f4ff; margin-bottom: 8px; } + #streaming-player { border: 1px solid var(--border-color-primary) !important; border-radius: var(--block-radius) !important; padding: var(--block-padding) !important; } """ - with gr.Blocks(css=css, title="Qwen3-TTS Demo") as demo: - gr.Markdown("# Qwen3-TTS Online Serving Demo") - gr.Markdown(f"**Server:** `{api_base}`") + theme = gr.themes.Default( + primary_hue=gr.themes.Color( + c50="#f0f5ff", + c100="#dce6f9", + c200="#b8cef3", + c300="#8eb2eb", + c400="#6496e0", + c500="#4A90D9", + c600="#3a7bc8", + c700="#2d66b0", + c800="#1f4f8f", + c900="#163a6e", + c950="#0e2650", + ), + ) + + with gr.Blocks( + css=css, + title="Qwen3-TTS Demo", + js=_build_player_js(PCM_SAMPLE_RATE), + theme=theme, + ) as demo: + gr.HTML(f""" +
+ vLLM-Omni +
+

Qwen3-TTS Streaming Demo

+ + Served by vLLM-Omni +  ·  {api_base} + +
+
+ """) with gr.Row(): - # Left column: inputs with gr.Column(scale=3): text_input = gr.Textbox( label="Text to Synthesize", placeholder="Enter text here, e.g., Hello, how are you?", lines=4, ) - with gr.Row(): task_type = gr.Radio( choices=TASK_TYPES, @@ -221,53 +484,49 @@ def build_interface(api_base: str, stream_chunk_seconds: float = 0.25): label="Language", scale=1, ) - voice = gr.Dropdown( choices=voices, value=voices[0] if voices else None, - label="Speaker", + label="speaker", visible=True, + allow_custom_value=True, ) - instructions = gr.Textbox( label="Instructions", - placeholder=("e.g., Speak with excitement / A warm, friendly female voice"), + placeholder="e.g., Speak with excitement / A warm, friendly female voice", lines=2, visible=True, info="Optional style/emotion instructions", ) - - ref_audio = gr.Audio( - label="Reference Audio (upload for voice cloning)", - type="numpy", - sources=["upload", "microphone"], - visible=False, - ) - ref_audio_url = gr.Textbox( - label="Reference Audio URL", - placeholder=("https://example.com/reference.wav (alternative to uploading)"), - lines=1, - visible=False, - ) - ref_text = gr.Textbox( - label="Reference Audio Transcript", - placeholder=("Transcript of the reference audio (optional, improves quality)"), - lines=2, - visible=False, - ) - x_vector_only = gr.Checkbox( - label="Use x-vector only", - value=False, - visible=False, - info=("Skip reference transcript, use speaker embedding only (lower quality)"), - ) + with gr.Column(visible=False) as ref_group: + ref_audio = gr.Audio( + label="Reference Audio (upload for voice cloning)", + type="numpy", + sources=["upload", "microphone"], + ) + ref_audio_url = gr.Textbox( + label="Reference Audio URL", + placeholder="https://example.com/reference.wav", + lines=1, + ) + ref_text = gr.Textbox( + label="Reference Audio Transcript", + placeholder="Transcript of reference audio (optional, improves quality)", + lines=2, + ) + x_vector_only = gr.Checkbox( + label="Use x-vector only", + value=False, + info="Skip reference transcript, use speaker embedding only", + ) with gr.Row(): response_format = gr.Dropdown( choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], - value="wav", + value="pcm", label="Audio Format", scale=1, + interactive=False, ) speed = gr.Slider( minimum=0.25, @@ -276,60 +535,179 @@ def build_interface(api_base: str, stream_chunk_seconds: float = 0.25): step=0.05, label="Speed", scale=1, + interactive=False, ) stream_checkbox = gr.Checkbox( - label="Stream output", - value=False, - info=( - "Enable streaming (PCM format, speed disabled). " - "For gapless streaming, use gradio_fastrtc_demo.py" - ), + label="Stream (gapless)", + value=True, + info="AudioWorklet streaming (recommended)", scale=1, ) - generate_btn = gr.Button( - "Generate Speech", - variant="primary", - size="lg", - elem_id="generate-btn", - ) + with gr.Row(): + generate_btn = gr.Button( + "Generate Speech", + variant="primary", + size="lg", + elem_id="generate-btn", + scale=3, + ) + reset_btn = gr.Button( + "Reset", + variant="secondary", + size="lg", + scale=1, + ) - # Right column: output with gr.Column(scale=2): + player_html = gr.HTML( + value=PLAYER_HTML, + visible=True, + label="streaming player", + elem_id="streaming-player", + ) audio_output = gr.Audio( - label="Generated Audio", + label="generated audio", interactive=False, - streaming=True, autoplay=True, + visible=False, ) - gr.Markdown( - "### Task Types\n" - "- **CustomVoice**: Use a predefined speaker " - "(Vivian, Ryan, etc.) with optional style instructions\n" - "- **VoiceDesign**: Describe the desired voice in natural " - "language (instructions required)\n" - "- **Base**: Clone a voice from reference audio " - "(upload a file or provide a URL)" - ) + with gr.Column(visible=True) as examples_cv: + gr.Examples( + examples=[ + [ + "Have you ever wondered what it would be like to travel through time and visit ancient civilizations? The possibilities are endless, from witnessing the construction of the pyramids to experiencing the Renaissance firsthand.", + "ryan", + "English", + "", + ], + [ + "其实我真的有发现，我是一个特别善于观察别人情绪的人。比如说在一个聚会上，我总能第一时间察觉到谁不太开心，然后想办法让大家都能融入到欢乐的氛围中来。", + "vivian", + "Chinese", + "用特别愤怒的语气说", + ], + [ + "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door. He set down his cup of tea, adjusted his glasses, and walked slowly toward the entrance.", + "aiden", + "English", + "Speak in a mysterious, suspenseful tone", + ], + ], + inputs=[text_input, voice, language, instructions], + label="examples", + ) + with gr.Column(visible=False) as examples_vd: + gr.Examples( + examples=[ + [ + "It's in the top drawer... wait, it's empty? No way, that's impossible! I'm sure I put it there! Someone must have moved it while I was gone.", + "English", + "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice.", + ], + [ + "哥哥，你回来啦，人家等了你好久好久了，要抱抱！你去哪里了呀，都不跟人家说一声，人家好担心你哦。", + "Chinese", + "体现撒娇稚嫩的萝莉女声，音调偏高且起伏明显，营造出黏人、做作又刻意卖萌的听觉效果。", + ], + ], + inputs=[text_input, language, instructions], + label="examples", + ) + with gr.Column(visible=False) as examples_base: + gr.Examples( + examples=[ + [ + "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye. I hope you enjoy the rest of your evening and have a wonderful time.", + "English", + "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav", + "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you.", + ], + [ + "其实我真的有发现，我是一个特别善于观察别人情绪的人。比如说在一个聚会上，我总能第一时间察觉到谁不太开心。", + "Chinese", + "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav", + "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you.", + ], + ], + inputs=[text_input, language, ref_audio_url, ref_text], + label="examples", + ) + gr.HTML(""" +
+ + vLLM-Omni + +
+ """) + + # Hidden textbox to pass payload from Python → JavaScript + hidden_payload = gr.Textbox(visible=False, elem_id="tts-payload") + + # ── Event wiring ───────────────────────────────────────── + def on_task_change(tt): + is_cv = tt == "CustomVoice" + is_vd = tt == "VoiceDesign" + is_base = tt == "Base" + return ( + gr.update(visible=is_cv), # voice + gr.update( + visible=(is_cv or is_vd), + info="Required: describe the voice style" if is_vd else "Optional style/emotion instructions", + ), # instructions + gr.update(visible=is_base), # ref_group + gr.update(visible=is_cv), # examples_cv + gr.update(visible=is_vd), # examples_vd + gr.update(visible=is_base), # examples_base + ) - # Dynamic UI updates task_type.change( - fn=on_task_type_change, + fn=on_task_change, inputs=[task_type], outputs=[ voice, instructions, - ref_audio, - ref_audio_url, - ref_text, - x_vector_only, + ref_group, + examples_cv, + examples_vd, + examples_base, ], ) + def on_stream_change(stream: bool): + if stream: + return ( + gr.update(value="pcm", interactive=False), + gr.update(interactive=False), + gr.update(visible=True), # player + gr.update(visible=False), # audio + ) + return ( + gr.update(value="wav", interactive=True), + gr.update(interactive=True), + gr.update(visible=False), + gr.update(visible=True), + ) + stream_checkbox.change( fn=on_stream_change, inputs=[stream_checkbox], - outputs=[response_format, speed], + outputs=[response_format, speed, player_html, audio_output], + ) + + def on_reset(): + return ( + "", # text + None, # audio_output + "", # hidden_payload + PLAYER_HTML, # reset player + ) + + reset_btn.click( + fn=on_reset, + outputs=[text_input, audio_output, hidden_payload, player_html], + js="() => { if (window.ttsStop) window.ttsStop(); }", ) all_inputs = [ @@ -346,58 +724,85 @@ def build_interface(api_base: str, stream_chunk_seconds: float = 0.25): speed, ] - def dispatch(stream_enabled, *args): + def on_generate(stream_enabled, *args): if stream_enabled: - yield from generate_speech_stream( - api_base, - *args, - chunk_seconds=stream_chunk_seconds, + import time as _time + + text, task_type_v, voice_v, lang_v, instr_v, ref_a, ref_url, ref_t, xvec, _fmt, _spd = args + # For streaming, use URL for ref_audio (base64 is too large + # for the Gradio textbox → JS → fetch pipeline). + # If user uploaded audio but no URL, encode and store server-side. + if ref_a is not None and not (ref_url and ref_url.strip()): + req_id = f"req-{int(_time.time() * 1000)}" + full_payload = build_payload( + text, + task_type_v, + voice_v, + lang_v, + instr_v, + ref_a, + ref_url, + ref_t, + xvec, + stream=True, + ) + _pending_payloads[req_id] = full_payload + browser_payload = {"_req_id": req_id, "_nonce": int(_time.time() * 1000)} + return json.dumps(browser_payload), gr.update() + + # URL-only path: payload is small, pass directly to browser + payload = build_payload( + text, + task_type_v, + voice_v, + lang_v, + instr_v, + None, + ref_url, + ref_t, + xvec, + stream=True, ) + payload["_nonce"] = int(_time.time() * 1000) + return json.dumps(payload), gr.update() else: - yield generate_speech(api_base, *args) + audio = generate_speech(api_base, *args) + return "", audio generate_btn.click( - fn=dispatch, + fn=on_generate, inputs=[stream_checkbox] + all_inputs, - outputs=[audio_output], + outputs=[hidden_payload, audio_output], + ).then( + fn=lambda p: p, + inputs=[hidden_payload], + outputs=[hidden_payload], + js="(p) => { if (p && p.trim()) { const d = JSON.parse(p); delete d._nonce; window.ttsGenerate(d); } return p; }", ) demo.queue() - return demo + + return gr.mount_gradio_app(fastapi_app, demo, path="/") def parse_args(): - parser = argparse.ArgumentParser(description="Gradio demo for Qwen3-TTS online serving.") - add_common_args(parser) - parser.add_argument( - "--stream-chunk-seconds", - type=float, - default=0.25, - help=( - "Seconds of audio to buffer per streaming chunk (default: 0.25). " - "Lower = smoother but more overhead, higher = chunkier." - ), + parser = argparse.ArgumentParser( + description="Gradio demo for Qwen3-TTS with gapless AudioWorklet streaming.", ) + add_common_args(parser) return parser.parse_args() def main(): args = parse_args() - + logging.basicConfig(level=logging.INFO) print(f"Connecting to vLLM server at: {args.api_base}") - demo = build_interface( - args.api_base, - stream_chunk_seconds=args.stream_chunk_seconds, - ) - try: - demo.launch( - server_name=args.host, - server_port=args.port, - share=args.share, - ) - except KeyboardInterrupt: - print("\nShutting down...") + app = create_app(args.api_base) + + import uvicorn + + uvicorn.run(app, host=args.host, port=args.port) if __name__ == "__main__": diff --git a/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py b/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py deleted file mode 100644 index a40ebfd89..000000000 --- a/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Gradio demo for Qwen3-TTS using FastRTC for gapless streaming audio. - -Uses WebRTC via FastRTC to deliver audio chunks as a continuous real-time -stream, avoiding the inter-chunk gaps present in Gradio's built-in streaming. - -Requires: pip install fastrtc - -Supports all 3 task types: - - CustomVoice: Predefined speaker with optional style instructions - - VoiceDesign: Natural language voice description - - Base: Voice cloning from reference audio (URL only in this demo) - -Usage: - # Start the server first (see run_server.sh), then: - python gradio_fastrtc_demo.py --api-base http://localhost:8000 - - # With public sharing: - python gradio_fastrtc_demo.py --api-base http://localhost:8000 --share -""" - -import argparse -import logging -from collections.abc import Generator - -import gradio as gr -import numpy as np -from fastrtc import WebRTC -from tts_common import ( - PCM_SAMPLE_RATE, - SUPPORTED_LANGUAGES, - TASK_TYPES, - add_common_args, - build_payload, - fetch_voices, - stream_pcm_chunks, -) - -logger = logging.getLogger(__name__) - - -def make_tts_generator(api_base: str): - """Create a generator function for FastRTC's mode='receive'. - - FastRTC calls this with the additional_inputs values whenever the user - clicks "Start Stream". It must be a synchronous generator yielding - (sample_rate, int16_ndarray) tuples. - """ - - def tts_generate( - text: str, - task_type: str, - voice: str, - language: str, - instructions: str, - ref_audio_url: str, - ref_text: str, - x_vector_only: bool, - ) -> Generator[tuple[int, np.ndarray], None, None]: - if not text or not text.strip(): - return - - payload = build_payload( - text, - task_type, - voice, - language, - instructions, - ref_audio=None, # WebRTC UI doesn't support file upload - ref_audio_url=ref_audio_url, - ref_text=ref_text, - x_vector_only=x_vector_only, - stream=True, - ) - - try: - for samples in stream_pcm_chunks(api_base, payload): - yield (PCM_SAMPLE_RATE, samples) - except Exception: - logger.exception("TTS streaming error") - - return tts_generate - - -def build_interface(api_base: str): - """Build a custom Gradio Blocks UI matching gradio_demo.py layout.""" - voices = fetch_voices(api_base) - handler = make_tts_generator(api_base) - - css = """ - #generate-btn button { width: 100%; } - .task-info { padding: 8px 12px; border-radius: 6px; - background: #f0f4ff; margin-bottom: 8px; } - """ - - with gr.Blocks(css=css, title="Qwen3-TTS (FastRTC)") as demo: - gr.Markdown("# Qwen3-TTS Online Serving Demo (FastRTC)") - gr.Markdown(f"**Server:** `{api_base}` | **Transport:** WebRTC (gapless streaming)") - - with gr.Row(): - # Left column: inputs - with gr.Column(scale=3): - text_input = gr.Textbox( - label="Text to Synthesize", - placeholder="Enter text here, e.g., Hello, how are you?", - lines=4, - ) - - with gr.Row(): - task_type = gr.Radio( - choices=TASK_TYPES, - value="CustomVoice", - label="Task Type", - scale=2, - ) - language = gr.Dropdown( - choices=SUPPORTED_LANGUAGES, - value="Auto", - label="Language", - scale=1, - ) - - voice = gr.Dropdown( - choices=voices, - value=voices[0] if voices else None, - label="Speaker", - visible=True, - ) - - instructions = gr.Textbox( - label="Instructions", - placeholder=("e.g., Speak with excitement / A warm, friendly female voice"), - lines=2, - visible=True, - info="Optional style/emotion instructions", - ) - - # Base (voice clone) controls - # Note: file upload is not supported with WebRTC, - # use a URL instead. - ref_audio_url = gr.Textbox( - label="Reference Audio URL", - placeholder=("https://example.com/reference.wav (alternative to uploading)"), - lines=1, - visible=False, - ) - ref_text = gr.Textbox( - label="Reference Audio Transcript", - placeholder=("Transcript of the reference audio (optional, improves quality)"), - lines=2, - visible=False, - ) - x_vector_only = gr.Checkbox( - label="Use x-vector only", - value=False, - visible=False, - info=("Skip reference transcript, use speaker embedding only (lower quality)"), - ) - - generate_btn = gr.Button( - "Generate Speech", - variant="primary", - size="lg", - elem_id="generate-btn", - ) - - # Right column: output - with gr.Column(scale=2): - webrtc_output = WebRTC( - label="Generated Audio (WebRTC)", - mode="receive", - modality="audio", - ) - gr.Markdown( - "### Task Types\n" - "- **CustomVoice**: Use a predefined speaker " - "(Vivian, Ryan, etc.) with optional style instructions\n" - "- **VoiceDesign**: Describe the desired voice in natural " - "language (instructions required)\n" - "- **Base**: Clone a voice from reference audio " - "(provide a URL)" - ) - - # Dynamic UI updates for task type - # FastRTC doesn't support file upload, so we only toggle - # 5 components (no ref_audio) - def on_task_type_change_fastrtc(task_type: str): - if task_type == "CustomVoice": - return ( - gr.update(visible=True), # voice - gr.update(visible=True, info="Optional style/emotion instructions"), - gr.update(visible=False), # ref_audio_url - gr.update(visible=False), # ref_text - gr.update(visible=False), # x_vector_only - ) - elif task_type == "VoiceDesign": - return ( - gr.update(visible=False), - gr.update(visible=True, info="Required: describe the voice style"), - gr.update(visible=False), - gr.update(visible=False), - gr.update(visible=False), - ) - elif task_type == "Base": - return ( - gr.update(visible=False), - gr.update(visible=False), - gr.update(visible=True), - gr.update(visible=True), - gr.update(visible=True), - ) - return ( - gr.update(visible=True), - gr.update(visible=True), - gr.update(visible=False), - gr.update(visible=False), - gr.update(visible=False), - ) - - task_type.change( - fn=on_task_type_change_fastrtc, - inputs=[task_type], - outputs=[ - voice, - instructions, - ref_audio_url, - ref_text, - x_vector_only, - ], - ) - - # Wire up WebRTC streaming - all_inputs = [ - text_input, - task_type, - voice, - language, - instructions, - ref_audio_url, - ref_text, - x_vector_only, - ] - - webrtc_output.stream( - fn=handler, - inputs=all_inputs, - outputs=[webrtc_output], - trigger=generate_btn.click, - ) - - demo.queue() - return demo - - -def parse_args(): - parser = argparse.ArgumentParser(description="Gradio FastRTC demo for Qwen3-TTS (gapless streaming).") - add_common_args(parser) - return parser.parse_args() - - -def main(): - args = parse_args() - logging.basicConfig(level=logging.INFO) - print(f"Connecting to vLLM server at: {args.api_base}") - - demo = build_interface(args.api_base) - demo.launch( - server_name=args.host, - server_port=args.port, - share=args.share, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/online_serving/qwen3_tts/run_gradio_demo.sh b/examples/online_serving/qwen3_tts/run_gradio_demo.sh index 1c7fe9e37..bcc0ddb7c 100644 --- a/examples/online_serving/qwen3_tts/run_gradio_demo.sh +++ b/examples/online_serving/qwen3_tts/run_gradio_demo.sh @@ -132,7 +132,6 @@ vllm-omni serve "$MODEL" \ --port "$SERVER_PORT" \ --gpu-memory-utilization 0.9 \ --trust-remote-code \ - --enforce-eager \ --omni 2>&1 | tee "$LOG_FILE" & SERVER_PID=$! @@ -180,7 +179,7 @@ fi echo "" echo "Starting Gradio demo..." cd "$SCRIPT_DIR" -GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT") +GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--host" "$GRADIO_IP" "--port" "$GRADIO_PORT") if [ "$GRADIO_SHARE" = true ]; then GRADIO_CMD+=("--share") fi diff --git a/examples/online_serving/qwen3_tts/run_server.sh b/examples/online_serving/qwen3_tts/run_server.sh index fd020f18b..6f4aa83a0 100755 --- a/examples/online_serving/qwen3_tts/run_server.sh +++ b/examples/online_serving/qwen3_tts/run_server.sh @@ -36,5 +36,4 @@ vllm-omni serve "$MODEL" \ --port 8091 \ --gpu-memory-utilization 0.9 \ --trust-remote-code \ - --enforce-eager \ --omni diff --git a/examples/online_serving/qwen3_tts/tts_common.py b/examples/online_serving/qwen3_tts/tts_common.py index 8e1c77c39..b31a75d80 100644 --- a/examples/online_serving/qwen3_tts/tts_common.py +++ b/examples/online_serving/qwen3_tts/tts_common.py @@ -39,7 +39,9 @@ def fetch_voices(api_base: str) -> list[str]: ) if resp.status_code == 200: data = resp.json() - return data.get("voices", ["Vivian", "Ryan"]) + voices = data.get("voices") or [] + if voices: + return voices except Exception: pass return ["Vivian", "Ryan"]