[Model] Add GLM-TTS text-to-speech model support (#3141)
Signed-off-by: BeatSeat <wendavid552@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
748470c1fb
commit
28ce618f0e
38 files changed
+7568
-7
No files matched your search
@@ -16,6 +16,7 @@ list of supported architectures across all modalities, see
|
||||
|---|---|---|---|---|---|---|
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
|
||||
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
|
||||
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
|
||||
@@ -92,6 +93,35 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
|
||||
|
||||
---
|
||||
|
||||
## GLM-TTS
|
||||
|
||||
2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
|
||||
--model zai-org/GLM-TTS \
|
||||
--text "你好,这是语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "这是参考音频的文本内容。" \
|
||||
--output-dir ./output
|
||||
```
|
||||
|
||||
### Architecture
|
||||
```
|
||||
Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
|
||||
(Llama-based) (32k vocab) (Flow Matching)
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
|
||||
- Reference audio should be 3-10 seconds.
|
||||
- First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
|
||||
- Default sampling: temperature=1.0, top_k=25, top_p=0.8 (RAS method).
|
||||
- The `--model` path should point to the repository root (not `llm/` subdirectory).
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-TTS End-to-End Offline Inference Example.
|
||||
|
||||
GLM-TTS is a two-stage TTS system:
|
||||
- Stage 0 (AR): Llama-based model generates speech tokens from text
|
||||
- Stage 1 (DiT): Flow matching model converts speech tokens to audio
|
||||
|
||||
Usage:
|
||||
# Sync two-stage (default)
|
||||
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
|
||||
--model /path/to/GLM-TTS \
|
||||
--text "你好,这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "参考音频的转录文本。" \
|
||||
--output-dir ./output
|
||||
|
||||
# Async chunk mode (streaming DiT)
|
||||
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
|
||||
--model /path/to/GLM-TTS --async-chunk \
|
||||
--text "你好,这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "参考音频的转录文本。" \
|
||||
--output-dir ./output
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.request import urlopen
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import yaml
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.glm_tts.glm_tts import build_glm_tts_prefill_metadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_DEPLOY_CONFIG = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"vllm_omni",
|
||||
"deploy",
|
||||
"glm_tts.yaml",
|
||||
)
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _load_ref_audio(ref_audio: str) -> tuple[torch.Tensor, int]:
|
||||
"""Load reference audio from file path, URL, or data URI."""
|
||||
if ref_audio.startswith(("http://", "https://")):
|
||||
with urlopen(ref_audio, timeout=60) as response:
|
||||
audio_obj: Any = io.BytesIO(response.read())
|
||||
elif ref_audio.startswith("data:"):
|
||||
_, _, encoded = ref_audio.partition(",")
|
||||
audio_obj = io.BytesIO(base64.b64decode(encoded))
|
||||
else:
|
||||
audio_obj = ref_audio
|
||||
wav_np, sr = sf.read(audio_obj, dtype="float32")
|
||||
if wav_np.ndim > 1:
|
||||
wav_np = wav_np.mean(axis=1)
|
||||
return torch.from_numpy(wav_np), int(sr)
|
||||
|
||||
|
||||
def _concat_audio(audio_val: Any) -> np.ndarray:
|
||||
"""Concatenate audio tensors from multimodal output."""
|
||||
if isinstance(audio_val, list):
|
||||
tensors = [torch.as_tensor(t).float().reshape(-1) for t in audio_val if t is not None]
|
||||
if not tensors:
|
||||
return np.zeros((0,), dtype=np.float32)
|
||||
return torch.cat(tensors, dim=-1).cpu().numpy().astype(np.float32, copy=False)
|
||||
if isinstance(audio_val, torch.Tensor):
|
||||
return audio_val.float().cpu().numpy().reshape(-1)
|
||||
return np.asarray(audio_val, dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
def _extract_sample_rate(audio_mm: dict) -> int:
|
||||
"""Extract sample rate from multimodal output dict."""
|
||||
sr_raw = audio_mm.get("sr", SAMPLE_RATE)
|
||||
if isinstance(sr_raw, list):
|
||||
sr_raw = sr_raw[-1] if sr_raw else SAMPLE_RATE
|
||||
if hasattr(sr_raw, "item"):
|
||||
return int(sr_raw.item())
|
||||
return int(sr_raw)
|
||||
|
||||
|
||||
def _modify_deploy_config(base_path: str, async_chunk: bool) -> str:
|
||||
"""Build deploy config with explicit sync/async mode and eager execution.
|
||||
|
||||
Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts.py``
|
||||
(``_get_deploy_config``) so that example runs match CI behavior.
|
||||
"""
|
||||
with open(base_path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
cfg["async_chunk"] = async_chunk
|
||||
for stage in cfg.get("stages", []):
|
||||
stage["enforce_eager"] = True
|
||||
if stage.get("stage_id") == 0:
|
||||
stage["async_scheduling"] = bool(async_chunk)
|
||||
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False, prefix="glm_tts_")
|
||||
yaml.dump(cfg, tmp)
|
||||
tmp.close()
|
||||
return tmp.name
|
||||
|
||||
|
||||
def main(args):
|
||||
"""Run offline GLM-TTS inference."""
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
base_deploy_config = args.deploy_config or DEFAULT_DEPLOY_CONFIG
|
||||
deploy_config_path = _modify_deploy_config(base_deploy_config, args.async_chunk)
|
||||
|
||||
ref_audio_wav, ref_audio_sr = _load_ref_audio(args.ref_audio)
|
||||
if not args.ref_text:
|
||||
raise ValueError("GLM-TTS requires --ref-audio and --ref-text.")
|
||||
|
||||
inputs = [
|
||||
{
|
||||
"prompt": args.text,
|
||||
"multi_modal_data": {
|
||||
"audio": (ref_audio_wav.float().cpu().numpy(), ref_audio_sr),
|
||||
},
|
||||
"modalities": ["audio"],
|
||||
"mm_processor_kwargs": {"prompt_text": args.ref_text},
|
||||
"additional_information": build_glm_tts_prefill_metadata(
|
||||
args.model,
|
||||
args.text,
|
||||
args.ref_text,
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
stage_configs_path=deploy_config_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
t_start = time.perf_counter()
|
||||
outputs = omni.generate(inputs)
|
||||
elapsed = (time.perf_counter() - t_start) * 1000
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, "No audio output found"
|
||||
|
||||
audio = _concat_audio(audio_mm["audio"])
|
||||
sr = _extract_sample_rate(audio_mm)
|
||||
out_path = os.path.join(args.output_dir, "output.wav")
|
||||
sf.write(out_path, audio, samplerate=sr, format="WAV")
|
||||
|
||||
logger.info("Saved %s (%.2fs @ %dHz)", out_path, len(audio) / sr, sr)
|
||||
logger.info("Total inference: %.1f ms", elapsed)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="GLM-TTS Text-to-Speech Example")
|
||||
parser.add_argument("--model", type=str, required=True, help="Model path")
|
||||
parser.add_argument("--text", type=str, default="你好,这是一个语音合成测试。")
|
||||
parser.add_argument("--output-dir", type=str, default="./output")
|
||||
parser.add_argument("--ref-audio", type=str, required=True, help="Reference WAV path/URL")
|
||||
parser.add_argument("--ref-text", type=str, required=True, help="Transcript of ref audio")
|
||||
parser.add_argument("--deploy-config", type=str, default=None)
|
||||
parser.add_argument(
|
||||
"--async-chunk",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable async_chunk mode (streaming DiT). Default: sync two-stage.",
|
||||
)
|
||||
parser.add_argument("--log-stats", action="store_true")
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=600)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
main(parse_args())
|
||||
@@ -15,6 +15,7 @@ For the full list of supported architectures across all modalities, see
|
||||
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|
||||
|---|---|---|---|---|---|
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
|
||||
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — |
|
||||
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
|
||||
@@ -94,6 +95,45 @@ For full request-shape documentation (all parameters, response formats, error co
|
||||
|
||||
---
|
||||
|
||||
## GLM-TTS
|
||||
|
||||
2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`.
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091
|
||||
# or:
|
||||
bash examples/online_serving/text_to_speech/glm_tts/run_server.sh /path/to/GLM-TTS
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
# Voice cloning (required)
|
||||
python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
|
||||
--text "你好,这是语音克隆测试。" \
|
||||
--ref-audio file:///path/to/ref.wav \
|
||||
--ref-text "这是参考音频的文本内容。"
|
||||
|
||||
# Custom format
|
||||
python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
|
||||
--text "Hello, this is a voice cloning test." \
|
||||
--ref-audio file:///path/to/ref.wav \
|
||||
--ref-text "Transcript of the reference audio." \
|
||||
--response-format mp3 -o output.mp3
|
||||
```
|
||||
|
||||
### Gradio demo
|
||||
```bash
|
||||
bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
|
||||
```
|
||||
|
||||
### Notes
|
||||
- Output: 24 kHz mono WAV via HiFT vocoder.
|
||||
- `ref_audio` + `ref_text` are **required** together on every request. Reference audio should be 3-10 seconds.
|
||||
- Voice cloning feature extraction (WhisperVQ, CampPlus, mel) runs on the model side — no external dependency on the serving layer.
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Gradio demo for GLM-TTS online serving via /v1/audio/speech API.
|
||||
|
||||
Supports:
|
||||
- Voice-cloned text-to-speech synthesis from reference audio (upload or URL)
|
||||
- Streaming (progressive PCM chunks) and non-streaming modes
|
||||
|
||||
Usage:
|
||||
# Start the server first (see run_server.sh), then:
|
||||
python gradio_demo.py --api-base http://localhost:8091
|
||||
|
||||
# Or use run_gradio_demo.sh to start both server and demo together.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
|
||||
try:
|
||||
import gradio as gr
|
||||
except ImportError:
|
||||
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
|
||||
import httpx
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
PCM_SAMPLE_RATE = 24000
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_data: tuple) -> str:
|
||||
"""Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
|
||||
sample_rate, audio_np = audio_data
|
||||
if audio_np.dtype != np.int16:
|
||||
if audio_np.dtype in (np.float32, np.float64):
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, audio_np, sample_rate, format="WAV")
|
||||
wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
return f"data:audio/wav;base64,{wav_b64}"
|
||||
|
||||
|
||||
def build_payload(
|
||||
text: str,
|
||||
ref_audio: tuple | None,
|
||||
ref_audio_url: str,
|
||||
ref_text: str,
|
||||
response_format: str = "wav",
|
||||
stream: bool = False,
|
||||
max_new_tokens: int = 4096,
|
||||
) -> dict:
|
||||
"""Build the /v1/audio/speech request payload."""
|
||||
if not text or not text.strip():
|
||||
raise gr.Error("Please enter text to synthesize.")
|
||||
|
||||
payload: dict = {
|
||||
"input": text.strip(),
|
||||
"voice": "default",
|
||||
"response_format": "pcm" if stream else response_format,
|
||||
"stream": stream,
|
||||
"max_new_tokens": max_new_tokens,
|
||||
}
|
||||
|
||||
# Voice cloning: ref_audio takes priority over URL.
|
||||
ref_url_stripped = ref_audio_url.strip() if ref_audio_url else ""
|
||||
if ref_audio is not None:
|
||||
payload["ref_audio"] = encode_audio_to_base64(ref_audio)
|
||||
elif ref_url_stripped:
|
||||
payload["ref_audio"] = ref_url_stripped
|
||||
else:
|
||||
raise gr.Error("GLM-TTS requires a reference audio file or URL.")
|
||||
|
||||
if not ref_text or not ref_text.strip():
|
||||
raise gr.Error("GLM-TTS requires a transcript of the reference audio.")
|
||||
payload["ref_text"] = ref_text.strip()
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens):
|
||||
"""Non-streaming: call /v1/audio/speech and return full audio."""
|
||||
payload = build_payload(
|
||||
text, ref_audio, ref_audio_url, ref_text, response_format, stream=False, max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
resp = client.post(
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.")
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
|
||||
content_type = resp.headers.get("content-type", "")
|
||||
if "application/json" in content_type:
|
||||
try:
|
||||
raise gr.Error(f"Server error: {resp.json()}")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
if response_format == "pcm":
|
||||
audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
|
||||
return (PCM_SAMPLE_RATE, audio_np)
|
||||
audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
return (sample_rate, audio_np.astype(np.float32))
|
||||
except Exception as e:
|
||||
raise gr.Error(f"Failed to decode audio: {e}")
|
||||
|
||||
|
||||
def generate_speech_stream(
|
||||
api_base, text, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens, chunk_seconds=0.5
|
||||
):
|
||||
"""Streaming: yield PCM chunks for progressive playback."""
|
||||
payload = build_payload(
|
||||
text, ref_audio, ref_audio_url, ref_text, response_format, stream=True, max_new_tokens=max_new_tokens
|
||||
)
|
||||
|
||||
min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
|
||||
prebuffer_chunks = 2
|
||||
pending: list[np.ndarray] = []
|
||||
pending_len = 0
|
||||
chunks_yielded = 0
|
||||
prebuffer: list[np.ndarray] = []
|
||||
|
||||
try:
|
||||
leftover = b""
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream(
|
||||
"POST",
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
) as resp:
|
||||
if resp.status_code != 200:
|
||||
resp.read()
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
for chunk in resp.iter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
raw = leftover + chunk
|
||||
usable = len(raw) - (len(raw) % 2)
|
||||
leftover = raw[usable:]
|
||||
if usable == 0:
|
||||
continue
|
||||
float_samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
|
||||
pending.append(float_samples)
|
||||
pending_len += len(float_samples)
|
||||
if pending_len >= min_chunk_samples:
|
||||
audio_chunk = np.concatenate(pending)
|
||||
pending.clear()
|
||||
pending_len = 0
|
||||
if chunks_yielded < prebuffer_chunks:
|
||||
prebuffer.append(audio_chunk)
|
||||
chunks_yielded += 1
|
||||
if chunks_yielded == prebuffer_chunks:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
|
||||
prebuffer.clear()
|
||||
else:
|
||||
yield (PCM_SAMPLE_RATE, audio_chunk)
|
||||
remaining = prebuffer + pending
|
||||
if remaining:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.")
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
|
||||
|
||||
|
||||
def on_stream_change(stream: bool):
|
||||
"""Lock format to PCM when streaming."""
|
||||
if stream:
|
||||
return gr.update(value="pcm", interactive=False)
|
||||
return gr.update(value="wav", interactive=True)
|
||||
|
||||
|
||||
def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
|
||||
"""Build the Gradio interface."""
|
||||
with gr.Blocks(title="GLM-TTS Demo") as demo:
|
||||
gr.Markdown("# GLM-TTS - Voice Cloning")
|
||||
gr.Markdown(f"**Server:** `{api_base}` | **Model:** GLM-TTS | **Output:** 24kHz")
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column(scale=3):
|
||||
text_input = gr.Textbox(
|
||||
label="Text to Synthesize",
|
||||
placeholder="Enter text here (supports Chinese and English)...",
|
||||
lines=4,
|
||||
)
|
||||
|
||||
with gr.Accordion("Reference Voice (required)", open=True):
|
||||
gr.Markdown("Upload or link a short reference audio (3-10s) and provide its transcript.")
|
||||
ref_audio = gr.Audio(
|
||||
label="Reference Audio",
|
||||
type="numpy",
|
||||
sources=["upload", "microphone"],
|
||||
)
|
||||
ref_audio_url = gr.Textbox(
|
||||
label="Reference Audio URL (alternative to upload)",
|
||||
placeholder="https://example.com/reference.wav",
|
||||
lines=1,
|
||||
)
|
||||
ref_text = gr.Textbox(
|
||||
label="Reference Audio Transcript (required for cloning)",
|
||||
placeholder="Exact transcript of the reference audio...",
|
||||
lines=2,
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
response_format = gr.Dropdown(
|
||||
choices=["wav", "mp3", "flac", "pcm"],
|
||||
value="wav",
|
||||
label="Audio Format",
|
||||
scale=1,
|
||||
)
|
||||
stream_checkbox = gr.Checkbox(
|
||||
label="Stream output",
|
||||
value=False,
|
||||
info="Progressive PCM streaming",
|
||||
scale=1,
|
||||
)
|
||||
max_new_tokens = gr.Slider(
|
||||
minimum=256,
|
||||
maximum=8192,
|
||||
value=4096,
|
||||
step=256,
|
||||
label="Max New Tokens",
|
||||
scale=1,
|
||||
)
|
||||
|
||||
generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
|
||||
|
||||
with gr.Column(scale=2):
|
||||
audio_output = gr.Audio(
|
||||
label="Generated Audio",
|
||||
interactive=False,
|
||||
streaming=True,
|
||||
autoplay=True,
|
||||
)
|
||||
gr.Markdown(
|
||||
"### About\n"
|
||||
"- **GLM-TTS** by Zhipu AI: Llama-based AR + Flow Matching DiT\n"
|
||||
"- **Voice cloning**: upload 3-10s reference + transcript\n"
|
||||
"- **Streaming**: real-time PCM output\n"
|
||||
"- **24kHz** output via Vocos vocoder"
|
||||
)
|
||||
|
||||
stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
|
||||
|
||||
all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens]
|
||||
|
||||
def dispatch(stream_enabled, *args):
|
||||
if stream_enabled:
|
||||
yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
|
||||
else:
|
||||
yield generate_speech(api_base, *args)
|
||||
|
||||
generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
|
||||
demo.queue()
|
||||
return demo
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Gradio demo for GLM-TTS")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)")
|
||||
parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)")
|
||||
parser.add_argument("--share", action="store_true", help="Share publicly")
|
||||
parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds per streaming chunk")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||
demo = build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds)
|
||||
demo.launch(server_name=args.host, server_port=args.port, share=args.share)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,173 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI-compatible client for GLM-TTS via /v1/audio/speech endpoint.
|
||||
|
||||
GLM-TTS is a two-stage TTS system (AR + DiT) that generates audio from text
|
||||
conditioned on reference speech. Each request requires ref_audio + ref_text.
|
||||
|
||||
Usage:
|
||||
# Voice cloning
|
||||
python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
|
||||
|
||||
# Streaming response, for async_chunk server mode
|
||||
python openai_speech_client.py --text "你好" --stream --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
|
||||
|
||||
# Specify output format
|
||||
python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav \
|
||||
--ref-text "参考文本" --response-format mp3 -o output.mp3
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import httpx
|
||||
|
||||
# Default server configuration
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
|
||||
|
||||
def run_tts_generation(args) -> None:
|
||||
"""Run TTS generation via OpenAI-compatible /v1/audio/speech API."""
|
||||
if not args.ref_audio or not args.ref_text:
|
||||
raise ValueError("GLM-TTS requires --ref-audio and --ref-text for voice cloning.")
|
||||
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"voice": "default",
|
||||
"input": args.text,
|
||||
"response_format": args.response_format,
|
||||
"stream": bool(args.stream),
|
||||
"ref_audio": args.ref_audio,
|
||||
"ref_text": args.ref_text,
|
||||
}
|
||||
if args.max_new_tokens:
|
||||
payload["max_new_tokens"] = args.max_new_tokens
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
|
||||
print(f"Stream: {args.stream}")
|
||||
print("Generating audio...")
|
||||
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "tts_output.pcm"
|
||||
with httpx.Client(timeout=300.0) as client, open(output_path, "wb") as f:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as response:
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
response.read()
|
||||
print(response.text)
|
||||
return
|
||||
for chunk in response.iter_bytes():
|
||||
f.write(chunk)
|
||||
print(f"Streaming audio saved to: {output_path}")
|
||||
else:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
output_path = args.output or f"tts_output.{args.response_format}"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="OpenAI-compatible client for GLM-TTS via /v1/audio/speech",
|
||||
)
|
||||
|
||||
# Server configuration
|
||||
parser.add_argument(
|
||||
"--api-base",
|
||||
type=str,
|
||||
default=DEFAULT_API_BASE,
|
||||
help=f"API base URL (default: {DEFAULT_API_BASE})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-key",
|
||||
type=str,
|
||||
default=DEFAULT_API_KEY,
|
||||
help="API key (default: EMPTY)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
"-m",
|
||||
type=str,
|
||||
default="glm-tts",
|
||||
help="Model name/path",
|
||||
)
|
||||
|
||||
# Input text
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Text to synthesize",
|
||||
)
|
||||
|
||||
# Generation parameters
|
||||
parser.add_argument(
|
||||
"--max-new-tokens",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Maximum new tokens to generate (default: model default)",
|
||||
)
|
||||
|
||||
# Output
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
type=str,
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio output format (default: wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
help="Request a streaming audio response (use with async_chunk server mode).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Output audio file path (default: tts_output.<format>)",
|
||||
)
|
||||
|
||||
# Voice cloning parameters
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Reference audio URL, file:// URI, or base64 data URL for voice cloning",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Transcript of the reference audio (required with --ref-audio)",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
run_tts_generation(args)
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Launch GLM-TTS server + Gradio demo together.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_gradio_demo.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-zai-org/GLM-TTS}"
|
||||
PORT="${PORT:-8091}"
|
||||
GRADIO_PORT="${GRADIO_PORT:-7860}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
echo "Starting GLM-TTS server (port $PORT)..."
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--deploy-config "$REPO_ROOT/vllm_omni/deploy/glm_tts.yaml" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
echo "Stopping server (PID $SERVER_PID)..."
|
||||
kill $SERVER_PID 2>/dev/null
|
||||
wait $SERVER_PID 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Wait for server to be ready.
|
||||
echo "Waiting for server to start..."
|
||||
for i in $(seq 1 120); do
|
||||
if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
|
||||
echo "Server ready."
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Starting Gradio demo (port $GRADIO_PORT)..."
|
||||
python "$SCRIPT_DIR/gradio_demo.py" \
|
||||
--api-base "http://localhost:$PORT" \
|
||||
--port "$GRADIO_PORT"
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for GLM-TTS models
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh # Default model path, async_chunk mode
|
||||
# ./run_server.sh /path/to/GLM-TTS # Custom model path, async_chunk mode
|
||||
# ./run_server.sh /path/to/GLM-TTS sync # Sync two-stage mode
|
||||
#
|
||||
# NOTE: The model path should point to the repo ROOT (not llm/ subdirectory).
|
||||
# model_subdir/tokenizer_subdir in the pipeline config resolve subdirectories.
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${1:-zai-org/GLM-TTS}"
|
||||
MODE="${2:-async}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
case "$MODE" in
|
||||
async|async_chunk)
|
||||
;;
|
||||
sync|no_async_chunk)
|
||||
EXTRA_ARGS+=("--no-async-chunk")
|
||||
;;
|
||||
*)
|
||||
echo "Unknown mode: $MODE (expected async or sync)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Starting GLM-TTS server with model: $MODEL (mode: $MODE)"
|
||||
|
||||
vllm-omni serve "$MODEL" \
|
||||
--deploy-config vllm_omni/deploy/glm_tts.yaml \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
--omni \
|
||||
"${EXTRA_ARGS[@]}"
|
||||
Reference in new issue
Block a user