[Model] Add GLM-TTS text-to-speech model support (#3141)

Signed-off-by: BeatSeat <wendavid552@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
BeatSeatandYueqian Lin authored and GitHub committed 2026-05-24 05:42:26 +08:00
1 parent 748470c1fb
commit 28ce618f0e
38 files changed
+7568 -7

No files matched your search

@@ -16,6 +16,7 @@ list of supported architectures across all modalities, see
|---|---|---|---|---|---|---|
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
| GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
@@ -92,6 +93,35 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
---
## GLM-TTS
2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
### Quick start
```bash
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
--model zai-org/GLM-TTS \
--text "你好,这是语音合成测试。" \
--ref-audio /path/to/reference.wav \
--ref-text "这是参考音频的文本内容。" \
--output-dir ./output
```
### Architecture
```
Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
(Llama-based) (32k vocab) (Flow Matching)
```
### Notes
- `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
- Reference audio should be 3-10 seconds.
- First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
- Default sampling: temperature=1.0, top_k=25, top_p=0.8 (RAS method).
- The `--model` path should point to the repository root (not `llm/` subdirectory).
---
## Fish Speech S2 Pro
4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-TTS End-to-End Offline Inference Example.
GLM-TTS is a two-stage TTS system:
- Stage 0 (AR): Llama-based model generates speech tokens from text
- Stage 1 (DiT): Flow matching model converts speech tokens to audio
Usage:
# Sync two-stage (default)
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
--model /path/to/GLM-TTS \
--text "你好,这是一个语音合成测试。" \
--ref-audio /path/to/reference.wav \
--ref-text "参考音频的转录文本。" \
--output-dir ./output
# Async chunk mode (streaming DiT)
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
--model /path/to/GLM-TTS --async-chunk \
--text "你好,这是一个语音合成测试。" \
--ref-audio /path/to/reference.wav \
--ref-text "参考音频的转录文本。" \
--output-dir ./output
"""
import base64
import io
import logging
import os
import tempfile
import time
from typing import Any
from urllib.request import urlopen
import numpy as np
import soundfile as sf
import torch
import yaml
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni import Omni
from vllm_omni.model_executor.models.glm_tts.glm_tts import build_glm_tts_prefill_metadata
logger = logging.getLogger(__name__)
DEFAULT_DEPLOY_CONFIG = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"vllm_omni",
"deploy",
"glm_tts.yaml",
)
SAMPLE_RATE = 24000
def _load_ref_audio(ref_audio: str) -> tuple[torch.Tensor, int]:
"""Load reference audio from file path, URL, or data URI."""
if ref_audio.startswith(("http://", "https://")):
with urlopen(ref_audio, timeout=60) as response:
audio_obj: Any = io.BytesIO(response.read())
elif ref_audio.startswith("data:"):
_, _, encoded = ref_audio.partition(",")
audio_obj = io.BytesIO(base64.b64decode(encoded))
else:
audio_obj = ref_audio
wav_np, sr = sf.read(audio_obj, dtype="float32")
if wav_np.ndim > 1:
wav_np = wav_np.mean(axis=1)
return torch.from_numpy(wav_np), int(sr)
def _concat_audio(audio_val: Any) -> np.ndarray:
"""Concatenate audio tensors from multimodal output."""
if isinstance(audio_val, list):
tensors = [torch.as_tensor(t).float().reshape(-1) for t in audio_val if t is not None]
if not tensors:
return np.zeros((0,), dtype=np.float32)
return torch.cat(tensors, dim=-1).cpu().numpy().astype(np.float32, copy=False)
if isinstance(audio_val, torch.Tensor):
return audio_val.float().cpu().numpy().reshape(-1)
return np.asarray(audio_val, dtype=np.float32).reshape(-1)
def _extract_sample_rate(audio_mm: dict) -> int:
"""Extract sample rate from multimodal output dict."""
sr_raw = audio_mm.get("sr", SAMPLE_RATE)
if isinstance(sr_raw, list):
sr_raw = sr_raw[-1] if sr_raw else SAMPLE_RATE
if hasattr(sr_raw, "item"):
return int(sr_raw.item())
return int(sr_raw)
def _modify_deploy_config(base_path: str, async_chunk: bool) -> str:
"""Build deploy config with explicit sync/async mode and eager execution.
Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts.py``
(``_get_deploy_config``) so that example runs match CI behavior.
"""
with open(base_path) as f:
cfg = yaml.safe_load(f)
cfg["async_chunk"] = async_chunk
for stage in cfg.get("stages", []):
stage["enforce_eager"] = True
if stage.get("stage_id") == 0:
stage["async_scheduling"] = bool(async_chunk)
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False, prefix="glm_tts_")
yaml.dump(cfg, tmp)
tmp.close()
return tmp.name
def main(args):
"""Run offline GLM-TTS inference."""
os.makedirs(args.output_dir, exist_ok=True)
base_deploy_config = args.deploy_config or DEFAULT_DEPLOY_CONFIG
deploy_config_path = _modify_deploy_config(base_deploy_config, args.async_chunk)
ref_audio_wav, ref_audio_sr = _load_ref_audio(args.ref_audio)
if not args.ref_text:
raise ValueError("GLM-TTS requires --ref-audio and --ref-text.")
inputs = [
{
"prompt": args.text,
"multi_modal_data": {
"audio": (ref_audio_wav.float().cpu().numpy(), ref_audio_sr),
},
"modalities": ["audio"],
"mm_processor_kwargs": {"prompt_text": args.ref_text},
"additional_information": build_glm_tts_prefill_metadata(
args.model,
args.text,
args.ref_text,
),
}
]
omni = Omni(
model=args.model,
stage_configs_path=deploy_config_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
)
t_start = time.perf_counter()
outputs = omni.generate(inputs)
elapsed = (time.perf_counter() - t_start) * 1000
assert outputs, "No outputs returned"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, "No audio output found"
audio = _concat_audio(audio_mm["audio"])
sr = _extract_sample_rate(audio_mm)
out_path = os.path.join(args.output_dir, "output.wav")
sf.write(out_path, audio, samplerate=sr, format="WAV")
logger.info("Saved %s (%.2fs @ %dHz)", out_path, len(audio) / sr, sr)
logger.info("Total inference: %.1f ms", elapsed)
def parse_args():
parser = FlexibleArgumentParser(description="GLM-TTS Text-to-Speech Example")
parser.add_argument("--model", type=str, required=True, help="Model path")
parser.add_argument("--text", type=str, default="你好,这是一个语音合成测试。")
parser.add_argument("--output-dir", type=str, default="./output")
parser.add_argument("--ref-audio", type=str, required=True, help="Reference WAV path/URL")
parser.add_argument("--ref-text", type=str, required=True, help="Transcript of ref audio")
parser.add_argument("--deploy-config", type=str, default=None)
parser.add_argument(
"--async-chunk",
action="store_true",
default=False,
help="Enable async_chunk mode (streaming DiT). Default: sync two-stage.",
)
parser.add_argument("--log-stats", action="store_true")
parser.add_argument("--stage-init-timeout", type=int, default=600)
return parser.parse_args()
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
main(parse_args())
@@ -15,6 +15,7 @@ For the full list of supported architectures across all modalities, see
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|---|---|---|---|---|---|
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — |
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
@@ -94,6 +95,45 @@ For full request-shape documentation (all parameters, response formats, error co
---
## GLM-TTS
2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`.
### Launch
```bash
vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091
# or:
bash examples/online_serving/text_to_speech/glm_tts/run_server.sh /path/to/GLM-TTS
```
### Sending requests
```bash
# Voice cloning (required)
python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
--text "你好,这是语音克隆测试。" \
--ref-audio file:///path/to/ref.wav \
--ref-text "这是参考音频的文本内容。"
# Custom format
python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
--text "Hello, this is a voice cloning test." \
--ref-audio file:///path/to/ref.wav \
--ref-text "Transcript of the reference audio." \
--response-format mp3 -o output.mp3
```
### Gradio demo
```bash
bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
```
### Notes
- Output: 24 kHz mono WAV via HiFT vocoder.
- `ref_audio` + `ref_text` are **required** together on every request. Reference audio should be 3-10 seconds.
- Voice cloning feature extraction (WhisperVQ, CampPlus, mel) runs on the model side — no external dependency on the serving layer.
---
## Fish Speech S2 Pro
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
@@ -0,0 +1,291 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gradio demo for GLM-TTS online serving via /v1/audio/speech API.
Supports:
- Voice-cloned text-to-speech synthesis from reference audio (upload or URL)
- Streaming (progressive PCM chunks) and non-streaming modes
Usage:
# Start the server first (see run_server.sh), then:
python gradio_demo.py --api-base http://localhost:8091
# Or use run_gradio_demo.sh to start both server and demo together.
"""
import argparse
import base64
import io
try:
import gradio as gr
except ImportError:
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
import httpx
import numpy as np
import soundfile as sf
PCM_SAMPLE_RATE = 24000
DEFAULT_API_BASE = "http://localhost:8091"
def encode_audio_to_base64(audio_data: tuple) -> str:
"""Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
sample_rate, audio_np = audio_data
if audio_np.dtype != np.int16:
if audio_np.dtype in (np.float32, np.float64):
audio_np = np.clip(audio_np, -1.0, 1.0)
audio_np = (audio_np * 32767).astype(np.int16)
else:
audio_np = audio_np.astype(np.int16)
buf = io.BytesIO()
sf.write(buf, audio_np, sample_rate, format="WAV")
wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
return f"data:audio/wav;base64,{wav_b64}"
def build_payload(
text: str,
ref_audio: tuple | None,
ref_audio_url: str,
ref_text: str,
response_format: str = "wav",
stream: bool = False,
max_new_tokens: int = 4096,
) -> dict:
"""Build the /v1/audio/speech request payload."""
if not text or not text.strip():
raise gr.Error("Please enter text to synthesize.")
payload: dict = {
"input": text.strip(),
"voice": "default",
"response_format": "pcm" if stream else response_format,
"stream": stream,
"max_new_tokens": max_new_tokens,
}
# Voice cloning: ref_audio takes priority over URL.
ref_url_stripped = ref_audio_url.strip() if ref_audio_url else ""
if ref_audio is not None:
payload["ref_audio"] = encode_audio_to_base64(ref_audio)
elif ref_url_stripped:
payload["ref_audio"] = ref_url_stripped
else:
raise gr.Error("GLM-TTS requires a reference audio file or URL.")
if not ref_text or not ref_text.strip():
raise gr.Error("GLM-TTS requires a transcript of the reference audio.")
payload["ref_text"] = ref_text.strip()
return payload
def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens):
"""Non-streaming: call /v1/audio/speech and return full audio."""
payload = build_payload(
text, ref_audio, ref_audio_url, ref_text, response_format, stream=False, max_new_tokens=max_new_tokens
)
try:
with httpx.Client(timeout=300.0) as client:
resp = client.post(
f"{api_base}/v1/audio/speech",
json=payload,
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
)
except httpx.TimeoutException:
raise gr.Error("Request timed out. The server may be busy.")
except httpx.ConnectError:
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
if resp.status_code != 200:
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
content_type = resp.headers.get("content-type", "")
if "application/json" in content_type:
try:
raise gr.Error(f"Server error: {resp.json()}")
except ValueError:
pass
try:
if response_format == "pcm":
audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
return (PCM_SAMPLE_RATE, audio_np)
audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
if audio_np.ndim > 1:
audio_np = audio_np[:, 0]
return (sample_rate, audio_np.astype(np.float32))
except Exception as e:
raise gr.Error(f"Failed to decode audio: {e}")
def generate_speech_stream(
api_base, text, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens, chunk_seconds=0.5
):
"""Streaming: yield PCM chunks for progressive playback."""
payload = build_payload(
text, ref_audio, ref_audio_url, ref_text, response_format, stream=True, max_new_tokens=max_new_tokens
)
min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
prebuffer_chunks = 2
pending: list[np.ndarray] = []
pending_len = 0
chunks_yielded = 0
prebuffer: list[np.ndarray] = []
try:
leftover = b""
with httpx.Client(timeout=300.0) as client:
with client.stream(
"POST",
f"{api_base}/v1/audio/speech",
json=payload,
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
) as resp:
if resp.status_code != 200:
resp.read()
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
for chunk in resp.iter_bytes():
if not chunk:
continue
raw = leftover + chunk
usable = len(raw) - (len(raw) % 2)
leftover = raw[usable:]
if usable == 0:
continue
float_samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
pending.append(float_samples)
pending_len += len(float_samples)
if pending_len >= min_chunk_samples:
audio_chunk = np.concatenate(pending)
pending.clear()
pending_len = 0
if chunks_yielded < prebuffer_chunks:
prebuffer.append(audio_chunk)
chunks_yielded += 1
if chunks_yielded == prebuffer_chunks:
yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
prebuffer.clear()
else:
yield (PCM_SAMPLE_RATE, audio_chunk)
remaining = prebuffer + pending
if remaining:
yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
except httpx.TimeoutException:
raise gr.Error("Request timed out. The server may be busy.")
except httpx.ConnectError:
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
def on_stream_change(stream: bool):
"""Lock format to PCM when streaming."""
if stream:
return gr.update(value="pcm", interactive=False)
return gr.update(value="wav", interactive=True)
def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
"""Build the Gradio interface."""
with gr.Blocks(title="GLM-TTS Demo") as demo:
gr.Markdown("# GLM-TTS - Voice Cloning")
gr.Markdown(f"**Server:** `{api_base}` | **Model:** GLM-TTS | **Output:** 24kHz")
with gr.Row():
with gr.Column(scale=3):
text_input = gr.Textbox(
label="Text to Synthesize",
placeholder="Enter text here (supports Chinese and English)...",
lines=4,
)
with gr.Accordion("Reference Voice (required)", open=True):
gr.Markdown("Upload or link a short reference audio (3-10s) and provide its transcript.")
ref_audio = gr.Audio(
label="Reference Audio",
type="numpy",
sources=["upload", "microphone"],
)
ref_audio_url = gr.Textbox(
label="Reference Audio URL (alternative to upload)",
placeholder="https://example.com/reference.wav",
lines=1,
)
ref_text = gr.Textbox(
label="Reference Audio Transcript (required for cloning)",
placeholder="Exact transcript of the reference audio...",
lines=2,
)
with gr.Row():
response_format = gr.Dropdown(
choices=["wav", "mp3", "flac", "pcm"],
value="wav",
label="Audio Format",
scale=1,
)
stream_checkbox = gr.Checkbox(
label="Stream output",
value=False,
info="Progressive PCM streaming",
scale=1,
)
max_new_tokens = gr.Slider(
minimum=256,
maximum=8192,
value=4096,
step=256,
label="Max New Tokens",
scale=1,
)
generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
with gr.Column(scale=2):
audio_output = gr.Audio(
label="Generated Audio",
interactive=False,
streaming=True,
autoplay=True,
)
gr.Markdown(
"### About\n"
"- **GLM-TTS** by Zhipu AI: Llama-based AR + Flow Matching DiT\n"
"- **Voice cloning**: upload 3-10s reference + transcript\n"
"- **Streaming**: real-time PCM output\n"
"- **24kHz** output via Vocos vocoder"
)
stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens]
def dispatch(stream_enabled, *args):
if stream_enabled:
yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
else:
yield generate_speech(api_base, *args)
generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
demo.queue()
return demo
def main():
parser = argparse.ArgumentParser(description="Gradio demo for GLM-TTS")
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)")
parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)")
parser.add_argument("--share", action="store_true", help="Share publicly")
parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds per streaming chunk")
args = parser.parse_args()
print(f"Connecting to vLLM server at: {args.api_base}")
demo = build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds)
demo.launch(server_name=args.host, server_port=args.port, share=args.share)
if __name__ == "__main__":
main()
@@ -0,0 +1,173 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OpenAI-compatible client for GLM-TTS via /v1/audio/speech endpoint.
GLM-TTS is a two-stage TTS system (AR + DiT) that generates audio from text
conditioned on reference speech. Each request requires ref_audio + ref_text.
Usage:
# Voice cloning
python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
# Streaming response, for async_chunk server mode
python openai_speech_client.py --text "你好" --stream --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
# Specify output format
python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav \
--ref-text "参考文本" --response-format mp3 -o output.mp3
"""
import argparse
import httpx
# Default server configuration
DEFAULT_API_BASE = "http://localhost:8091"
DEFAULT_API_KEY = "EMPTY"
def run_tts_generation(args) -> None:
"""Run TTS generation via OpenAI-compatible /v1/audio/speech API."""
if not args.ref_audio or not args.ref_text:
raise ValueError("GLM-TTS requires --ref-audio and --ref-text for voice cloning.")
payload = {
"model": args.model,
"voice": "default",
"input": args.text,
"response_format": args.response_format,
"stream": bool(args.stream),
"ref_audio": args.ref_audio,
"ref_text": args.ref_text,
}
if args.max_new_tokens:
payload["max_new_tokens"] = args.max_new_tokens
print(f"Model: {args.model}")
print(f"Text: {args.text}")
print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
print(f"Stream: {args.stream}")
print("Generating audio...")
api_url = f"{args.api_base}/v1/audio/speech"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {args.api_key}",
}
if args.stream:
output_path = args.output or "tts_output.pcm"
with httpx.Client(timeout=300.0) as client, open(output_path, "wb") as f:
with client.stream("POST", api_url, json=payload, headers=headers) as response:
if response.status_code != 200:
print(f"Error: {response.status_code}")
response.read()
print(response.text)
return
for chunk in response.iter_bytes():
f.write(chunk)
print(f"Streaming audio saved to: {output_path}")
else:
with httpx.Client(timeout=300.0) as client:
response = client.post(api_url, json=payload, headers=headers)
if response.status_code != 200:
print(f"Error: {response.status_code}")
print(response.text)
return
try:
text = response.content.decode("utf-8")
if text.startswith('{"error"'):
print(f"Error: {text}")
return
except UnicodeDecodeError:
pass
output_path = args.output or f"tts_output.{args.response_format}"
with open(output_path, "wb") as f:
f.write(response.content)
print(f"Audio saved to: {output_path}")
def parse_args():
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description="OpenAI-compatible client for GLM-TTS via /v1/audio/speech",
)
# Server configuration
parser.add_argument(
"--api-base",
type=str,
default=DEFAULT_API_BASE,
help=f"API base URL (default: {DEFAULT_API_BASE})",
)
parser.add_argument(
"--api-key",
type=str,
default=DEFAULT_API_KEY,
help="API key (default: EMPTY)",
)
parser.add_argument(
"--model",
"-m",
type=str,
default="glm-tts",
help="Model name/path",
)
# Input text
parser.add_argument(
"--text",
type=str,
required=True,
help="Text to synthesize",
)
# Generation parameters
parser.add_argument(
"--max-new-tokens",
type=int,
default=None,
help="Maximum new tokens to generate (default: model default)",
)
# Output
parser.add_argument(
"--response-format",
type=str,
default="wav",
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
help="Audio output format (default: wav)",
)
parser.add_argument(
"--stream",
action="store_true",
help="Request a streaming audio response (use with async_chunk server mode).",
)
parser.add_argument(
"--output",
"-o",
type=str,
default=None,
help="Output audio file path (default: tts_output.<format>)",
)
# Voice cloning parameters
parser.add_argument(
"--ref-audio",
type=str,
default=None,
help="Reference audio URL, file:// URI, or base64 data URL for voice cloning",
)
parser.add_argument(
"--ref-text",
type=str,
default=None,
help="Transcript of the reference audio (required with --ref-audio)",
)
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
run_tts_generation(args)
@@ -0,0 +1,48 @@
#!/bin/bash
# Launch GLM-TTS server + Gradio demo together.
#
# Usage:
# ./run_gradio_demo.sh
# CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh
set -e
MODEL="${MODEL:-zai-org/GLM-TTS}"
PORT="${PORT:-8091}"
GRADIO_PORT="${GRADIO_PORT:-7860}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
echo "Starting GLM-TTS server (port $PORT)..."
FLASHINFER_DISABLE_VERSION_CHECK=1 \
vllm-omni serve "$MODEL" \
--deploy-config "$REPO_ROOT/vllm_omni/deploy/glm_tts.yaml" \
--host 0.0.0.0 \
--port "$PORT" \
--gpu-memory-utilization 0.9 \
--trust-remote-code \
--enforce-eager \
--omni &
SERVER_PID=$!
cleanup() {
echo "Stopping server (PID $SERVER_PID)..."
kill $SERVER_PID 2>/dev/null
wait $SERVER_PID 2>/dev/null
}
trap cleanup EXIT
# Wait for server to be ready.
echo "Waiting for server to start..."
for i in $(seq 1 120); do
if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
echo "Server ready."
break
fi
sleep 2
done
echo "Starting Gradio demo (port $GRADIO_PORT)..."
python "$SCRIPT_DIR/gradio_demo.py" \
--api-base "http://localhost:$PORT" \
--port "$GRADIO_PORT"
@@ -0,0 +1,38 @@
#!/bin/bash
# Launch vLLM-Omni server for GLM-TTS models
#
# Usage:
# ./run_server.sh # Default model path, async_chunk mode
# ./run_server.sh /path/to/GLM-TTS # Custom model path, async_chunk mode
# ./run_server.sh /path/to/GLM-TTS sync # Sync two-stage mode
#
# NOTE: The model path should point to the repo ROOT (not llm/ subdirectory).
# model_subdir/tokenizer_subdir in the pipeline config resolve subdirectories.
set -e
MODEL="${1:-zai-org/GLM-TTS}"
MODE="${2:-async}"
EXTRA_ARGS=()
case "$MODE" in
async|async_chunk)
;;
sync|no_async_chunk)
EXTRA_ARGS+=("--no-async-chunk")
;;
*)
echo "Unknown mode: $MODE (expected async or sync)" >&2
exit 1
;;
esac
echo "Starting GLM-TTS server with model: $MODEL (mode: $MODE)"
vllm-omni serve "$MODEL" \
--deploy-config vllm_omni/deploy/glm_tts.yaml \
--host 0.0.0.0 \
--port 8091 \
--trust-remote-code \
--omni \
"${EXTRA_ARGS[@]}"