[Maintenance] Refresh MOSS-TTS optimization on main

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-09 12:46:57 +08:00
commit 28dd49a541
334 files changed
+24909 -6187

No files matched your search

@@ -5,10 +5,12 @@ This example shows how to use vLLM-Omni for running offline inference
with the correct prompt format on Qwen2.5-Omni
"""
import argparse
import functools
import json
import os
import time
from typing import NamedTuple
from typing import Any, NamedTuple
import numpy as np
import soundfile as sf
@@ -26,6 +28,20 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser
SEED = 42
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
try:
config = json.loads(value)
except json.JSONDecodeError as e:
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
if not isinstance(config, dict):
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
return config
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
class QueryResult(NamedTuple):
inputs: dict
limit_mm_per_prompt: dict[str, int]
@@ -378,11 +394,10 @@ def main(args):
for i, prompt in enumerate(prompts):
prompt["modalities"] = output_modalities
profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
if profiler_enabled and hasattr(omni, "start_profile"):
profiler_enabled = args.profiler_config is not None
if profiler_enabled:
print("[Profiler] Starting profiling...")
omni.start_profile(stages=[0])
elif profiler_enabled:
print("[Warn] VLLM_TORCH_PROFILER_DIR is set, but current engine does not support profiler controls.")
omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
# Determine output directory: prefer --output-dir; fallback to --output-wav
@@ -418,7 +433,7 @@ def main(args):
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
processed_count += 1
if profiler_enabled and hasattr(omni, "stop_profile") and processed_count >= total_requests:
if profiler_enabled and processed_count >= total_requests:
print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
# Stop the profiler while workers are still alive
omni.stop_profile()
@@ -476,6 +491,12 @@ def parse_args():
default=300,
help="Timeout for initializing stages in seconds (default: 300)",
)
parser.add_argument(
"--profiler-config",
type=parse_profiler_config,
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
parser.add_argument(
"--shm-threshold-bytes",
type=int,
@@ -14,6 +14,7 @@ list of supported architectures across all modalities, see
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|---|---|---|---|---|---|---|
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz |
@@ -44,6 +45,57 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \
---
## Audio8 TTS Preview
0.6B DualAR text-to-speech model (Audio8) with its own 44.1 kHz neural audio
codec: a slow AR transformer predicts one semantic token per codec frame, and a
4-layer fast AR predicts that frame's remaining 9 codebooks.
### Prerequisites
No extra packages: the codec architecture ships in tree and its weights
(`codec.pth`) come with the checkpoint.
### Quick start
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
```
### Voice cloning
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS." \
--ref-audio /path/to/reference.wav \
--ref-text "The exact transcript of the reference recording."
```
The reference transcript must match what is actually spoken in the clip;
a mismatch degrades both stability and speaker similarity.
### Streaming
```bash
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
--text "Welcome to Audio8 TTS." --streaming
```
### Notes
- Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
- Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
- Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
ends the utterance almost immediately. Keep the deploy defaults
(`temperature 0.7`, `top_k 50`, `top_p 0.9`), which also match
`generation_config.json`.
- Repetition-Aware Sampling is applied to the semantic token: a repeat inside
the last 10 frames is re-drawn from a flatter distribution instead of being
masked out.
- `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
### Measured behaviour (1x H20, shared GPU)
`--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
smoke-test magnitudes, not a benchmark.
---
## CosyVoice3
2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
@@ -130,11 +182,7 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
## Fish Speech S2 Pro
4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
### Prerequisites
```bash
pip install fish-speech
```
No extra packages are required; the DAC codec is vendored in vLLM-Omni.
### Quick start
```bash
@@ -0,0 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Offline inference demo for Audio8 TTS Preview 0.6B via vLLM Omni.
Usage:
# Text-only synthesis
python end2end.py --text "Welcome to Audio8 TTS."
# Zero-shot voice cloning (the transcript must match the reference audio)
python end2end.py --text "Welcome to Audio8 TTS." \
--ref-audio reference.wav --ref-text "The exact transcript of the reference recording."
# Streaming (per-chunk TTFA / inter-chunk timings)
python end2end.py --text "Welcome to Audio8 TTS." --streaming
"""
import asyncio
import os
import time
import numpy as np
import soundfile as sf
import torch
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from vllm_omni import AsyncOmni, Omni
from vllm_omni.model_executor.models.audio8_tts.codec_utils import estimate_reference_code_frames
from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
build_text_only_prompt_ids,
estimate_voice_clone_prompt_len,
normalize_text,
)
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
def build_prompt(
text: str,
ref_audio_path: str | None = None,
ref_text: str | None = None,
model_name: str = DEFAULT_MODEL,
) -> dict:
"""Build an engine prompt, using the same protocol as online serving."""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name)
if ref_audio_path is None and ref_text is None:
prompt_ids, normalized_text = build_text_only_prompt_ids(tokenizer, text)
return {
"prompt_token_ids": prompt_ids,
"additional_information": {"text": [normalized_text]},
}
if not ref_audio_path or not ref_text:
raise ValueError("Audio8 TTS voice cloning requires both --ref-audio and --ref-text")
normalized_text = normalize_text(text)
normalized_ref_text = normalize_text(ref_text, add_default_speaker=True)
ref_audio_wav, ref_audio_sr = sf.read(ref_audio_path, dtype="float32", always_2d=False)
if ref_audio_wav.ndim > 1:
ref_audio_wav = ref_audio_wav.mean(axis=-1)
ref_frames = estimate_reference_code_frames(int(ref_audio_wav.shape[0]), int(ref_audio_sr))
placeholder_len = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref_text, ref_frames)
# The clone prompt embeds the reference audio's own codec codes, so the
# model builds it in preprocess(); we only reserve the exact length here.
return {
"prompt_token_ids": [1] * placeholder_len,
"additional_information": {
"text": normalized_text,
"ref_text": normalized_ref_text,
"ref_audio_wav": torch.from_numpy(np.asarray(ref_audio_wav, dtype=np.float32)),
"ref_audio_sr": int(ref_audio_sr),
"audio8_structured_voice_clone": True,
},
}
def extract_audio(multimodal_output: dict) -> tuple[torch.Tensor, int]:
"""Pull the waveform and sample rate out of a multimodal output.
The output processor concatenates the per-step delta tensors under
``model_outputs``; ``audio`` is the older alias. Never use ``a or b`` on
these values -- truthiness on a tensor raises.
"""
audio = multimodal_output.get("model_outputs")
if audio is None:
audio = multimodal_output.get("audio")
if audio is None:
raise ValueError(f"No audio in multimodal_output: {sorted(multimodal_output)}")
if isinstance(audio, list):
audio = torch.cat([torch.as_tensor(chunk).reshape(-1) for chunk in audio], dim=0)
audio = torch.as_tensor(audio).reshape(-1)
sr_raw = multimodal_output.get("sr")
if isinstance(sr_raw, list):
sr_raw = sr_raw[-1] if sr_raw else None
if sr_raw is None:
raise ValueError("Missing sample rate in multimodal_output")
sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
return audio, sample_rate
def _save_wav(output_dir: str, request_id: str, audio: torch.Tensor, sample_rate: int) -> None:
out_wav = os.path.join(output_dir, f"output_{request_id}.wav")
sf.write(out_wav, audio.float().cpu().numpy(), samplerate=sample_rate, format="WAV")
duration = audio.numel() / sample_rate
# print(), not logging: vLLM reconfigures logging on engine start and
# disables loggers created before that, which would swallow these lines.
print(f"Request {request_id}: saved {out_wav} (sr={sample_rate}, {duration:.2f}s)")
def _omni_kwargs(args, model_name: str) -> dict:
kwargs = {
"model": model_name,
"log_stats": args.log_stats,
"stage_init_timeout": args.stage_init_timeout,
}
if args.deploy_config:
kwargs["deploy_config"] = args.deploy_config
return kwargs
def main(args) -> None:
os.makedirs(args.output_dir, exist_ok=True)
model_name = args.model or DEFAULT_MODEL
prompts = [
build_prompt(text, args.ref_audio, args.ref_text, model_name) for text in ([args.text] * args.num_prompts)
]
omni = Omni(**_omni_kwargs(args, model_name))
t_start = time.perf_counter()
total_audio_seconds = 0.0
for stage_outputs in omni.generate(prompts):
request_output = stage_outputs
if request_output is None or not request_output.outputs:
continue
audio, sample_rate = extract_audio(request_output.outputs[0].multimodal_output)
assert audio.numel() > 0, "Audio8 TTS produced an empty waveform"
total_audio_seconds += audio.numel() / sample_rate
_save_wav(args.output_dir, request_output.request_id, audio, sample_rate)
elapsed = time.perf_counter() - t_start
rtf = elapsed / max(total_audio_seconds, 1e-6)
print(f"Total {elapsed * 1000:.1f} ms for {total_audio_seconds:.2f} s of audio (RTF={rtf:.3f})")
async def main_streaming(args) -> None:
os.makedirs(args.output_dir, exist_ok=True)
model_name = args.model or DEFAULT_MODEL
prompt = build_prompt(args.text, args.ref_audio, args.ref_text, model_name)
omni = AsyncOmni(**_omni_kwargs(args, model_name))
request_id = "0"
chunks: list[torch.Tensor] = []
sample_rate = None
t_start = time.perf_counter()
t_prev = t_start
chunk_index = 0
async for stage_output in omni.generate(prompt, request_id=request_id):
multimodal_output = stage_output.outputs[0].multimodal_output
if not multimodal_output:
continue
audio = multimodal_output.get("model_outputs")
if audio is None:
audio = multimodal_output.get("audio")
if audio is not None:
if isinstance(audio, list):
chunks.extend(torch.as_tensor(chunk).reshape(-1) for chunk in audio)
else:
chunks.append(torch.as_tensor(audio).reshape(-1))
sr_raw = multimodal_output.get("sr")
if isinstance(sr_raw, list):
sr_raw = sr_raw[-1] if sr_raw else None
if sr_raw is not None:
sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
if stage_output.finished:
break
now = time.perf_counter()
if chunk_index == 0:
print(f"chunk 0: TTFA={(now - t_start) * 1000:.1f} ms")
else:
print(f"chunk {chunk_index}: inter_chunk={(now - t_prev) * 1000:.1f} ms")
t_prev = now
chunk_index += 1
if not chunks or sample_rate is None:
raise RuntimeError("Audio8 TTS streaming produced no audio")
_save_wav(args.output_dir, request_id, torch.cat(chunks, dim=0), sample_rate)
print(f"Streaming done: total={(time.perf_counter() - t_start) * 1000:.1f} ms chunks={chunk_index}")
def parse_args():
parser = TrackingArgumentParser(description="Audio8 TTS Preview offline inference with vLLM Omni")
parser.add_argument("--model", type=str, default=DEFAULT_MODEL, help="Model path or HuggingFace repo ID.")
parser.add_argument(
"--text",
type=str,
default="Welcome to Audio8 TTS, a compact multilingual text to speech model.",
help="Text to synthesize.",
)
parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
parser.add_argument(
"--ref-text",
type=str,
default=None,
help="Transcript of --ref-audio; required for voice cloning and must match the recording.",
)
parser.add_argument("--output-dir", type=str, default="output_audio", help="Directory for output WAV files.")
parser.add_argument(
"--num-prompts",
type=int,
default=1,
help="Duplicate the prompt N times to smoke-test concurrent requests.",
)
parser.add_argument(
"--deploy-config",
type=str,
default=None,
help="Override the deploy config path; defaults to vllm_omni/deploy/audio8_tts.yaml via model_type.",
)
parser.add_argument("--streaming", action="store_true", help="Use AsyncOmni and report per-chunk latency.")
parser.add_argument("--log-stats", action="store_true", help="Enable engine stats logging.")
parser.add_argument("--stage-init-timeout", type=int, default=600, help="Per-stage init timeout in seconds.")
return parser.parse_args()
if __name__ == "__main__":
parsed_args = parse_args()
if parsed_args.streaming:
asyncio.run(main_streaming(parsed_args))
else:
main(parsed_args)
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import functools
import json
import os
import urllib.request
from pathlib import Path
from typing import Any
import numpy as np
import soundfile as sf
@@ -29,6 +32,20 @@ def _default_ref_audio() -> str:
return str(dest)
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
try:
config = json.loads(value)
except json.JSONDecodeError as e:
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
if not isinstance(config, dict):
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
return config
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
def run_e2e():
parser = argparse.ArgumentParser()
# ""FunAudioLLM/Fun-CosyVoice3-0.5B-2512
@@ -64,6 +81,12 @@ def run_e2e():
required=True,
help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
)
parser.add_argument(
"--profiler-config",
type=parse_profiler_config,
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
args = parser.parse_args()
# Ensure tokenizer directory exists
if not os.path.exists(args.tokenizer):
@@ -79,6 +102,7 @@ def run_e2e():
deploy_config=args.deploy_config,
tokenizer=args.tokenizer,
log_stats=True,
profiler_config=args.profiler_config,
)
sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
@@ -149,8 +173,8 @@ def run_e2e():
sampling_params_list = [gpt_sampling, s2mel_sampling]
# Start profiling (requires VLLM_TORCH_PROFILER_DIR env var)
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
profiler_enabled = args.profiler_config is not None
if profiler_enabled:
print("Starting profiler...")
omni.start_profile()
@@ -158,7 +182,7 @@ def run_e2e():
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list[:2]))
# Stop profiling and get results
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
if profiler_enabled:
print("Stopping profiler...")
profile_results = omni.stop_profile()
print(f"Profile traces saved to: {profile_results}")
+21 -3
View File
@@ -264,7 +264,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
#### Realtime WebSocket client (`openai_realtime_client.py`)
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and receives **`response.audio.delta`** (incremental PCM for the reply) plus **`transcription.*`** events. By default it concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.
@@ -282,6 +282,7 @@ python openai_realtime_client.py \
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
--input-wav /path/to/input_16k_mono.wav \
--output-wav realtime_output.wav \
--server-vad \
--delta-dump-dir ./rt_delta_wavs
```
@@ -297,15 +298,32 @@ python openai_realtime_client.py \
| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
| `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits |
| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
Ensure the server is running, for example:
Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
```yaml
duplex_session:
server_vad_model_path: /models/silero_vad.onnx
```
Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
--omni \
--port 8091 \
--deploy-config /path/to/qwen3_omni_server_vad.yaml
```
Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
wire flow; `?duplex=1` remains a supported compatibility alias.
The Python client supports the following command-line arguments:
- `--query-type` (or `-q`): Query type (default: `use_video`). Options: `text`, `use_audio`, `use_image`, `use_video`
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
This client:
1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
3) Receives response.audio.* and transcription.* events,
3) Receives response.audio.*, response.output_text.*, and transcript events,
4) Saves synthesized audio to an output WAV file and optional text file.
By default each ``response.audio.delta`` is treated as an **incremental PCM**
@@ -18,6 +21,7 @@ Usage:
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
--input-wav input_16k_mono.wav \
--output-wav realtime_output.wav \
--server-vad \
--delta-dump-dir ./rt_delta_wavs
Dependencies:
@@ -32,6 +36,7 @@ import base64
import json
import wave
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
try:
import websockets
@@ -70,6 +75,14 @@ def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> Non
wf.writeframes(pcm16_bytes)
def _with_realtime_route(url: str, *, server_vad: bool) -> str:
parts = urlsplit(url)
query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "duplex"]
if not server_vad:
query.append(("duplex", "0"))
return urlunsplit(parts._replace(query=urlencode(query)))
async def run_client(
url: str,
model: str,
@@ -79,6 +92,7 @@ async def run_client(
chunk_ms: int,
send_delay_ms: int,
delta_dump_dir: Path | None,
server_vad: bool,
request_idx: int = 1,
total_requests: int = 1,
) -> None:
@@ -92,23 +106,44 @@ async def run_client(
delta_index = 0
text_chunks: list[str] = []
final_text: str = ""
final_audio_transcript: str = ""
if delta_dump_dir is not None:
delta_dump_dir.mkdir(parents=True, exist_ok=True)
# Keep the CLI mode authoritative even if the input URL already contains
# an explicit duplex query parameter.
url = _with_realtime_route(url, server_vad=server_vad)
async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
# 1) Validate model.
await ws.send(
json.dumps(
{
"type": "session.update",
if server_vad:
session_update = {
"type": "session.update",
"session": {
"model": model,
}
)
)
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16_000},
"turn_detection": {
"type": "server_vad",
"threshold": 0.5,
"prefix_padding_ms": 300,
"silence_duration_ms": 500,
"create_response": True,
"interrupt_response": False,
},
}
},
},
}
else:
# Preserve the existing non-duplex wire format.
session_update = {"type": "session.update", "model": model}
await ws.send(json.dumps(session_update))
# 2) Start generation once (non-final commit).
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
# 2) Preserve the legacy explicit-commit flow unless Server VAD owns endpointing.
if not server_vad:
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
# 3) Stream audio chunks.
for i in range(0, len(pcm16), chunk_bytes):
@@ -124,8 +159,20 @@ async def run_client(
if send_delay_ms > 0:
await asyncio.sleep(send_delay_ms / 1000.0)
# 4) Final commit closes input stream.
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
# 4) Finalize the turn explicitly, or append silence for Server VAD endpointing.
if server_vad:
trailing_silence = bytes(16_000 * 2)
for i in range(0, len(trailing_silence), chunk_bytes):
await ws.send(
json.dumps(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(trailing_silence[i : i + chunk_bytes]).decode("utf-8"),
}
)
)
else:
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
# 5) Receive server events until audio done.
while True:
@@ -144,7 +191,7 @@ async def run_client(
sr = event.get("sample_rate_hz")
if isinstance(sr, int) and sr > 0:
output_sample_rate = sr
audio_b64 = event.get("audio", "")
audio_b64 = event.get("delta" if server_vad else "audio", "")
if audio_b64:
pcm_delta = base64.b64decode(audio_b64)
incremental_pcm_parts.append(pcm_delta)
@@ -158,14 +205,14 @@ async def run_client(
)
continue
if event_type == "transcription.delta":
if event_type in {"response.output_text.delta", "transcription.delta"}:
delta = event.get("delta", "")
if delta:
text_chunks.append(delta)
print(delta, end="", flush=True)
continue
if event_type == "transcription.done":
if event_type in {"response.output_text.done", "transcription.done"}:
final_text = event.get("text", "") or "".join(text_chunks)
usage = event.get("usage")
final_text_with_tag = f"Final transcription: {final_text}"
@@ -176,7 +223,16 @@ async def run_client(
print(f"{log_prefix}text usage: {usage}")
continue
if event_type == "response.audio_transcript.done":
final_audio_transcript = event.get("transcript", "")
continue
if event_type == "response.audio.done":
if not server_vad:
break
continue
if event_type == "response.done" and server_vad:
break
if event_type == "error":
@@ -191,7 +247,7 @@ async def run_client(
print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
if output_text is not None:
text_to_save = final_text if final_text else "".join(text_chunks)
text_to_save = final_text or "".join(text_chunks) or final_audio_transcript
output_text.parent.mkdir(parents=True, exist_ok=True)
output_text.write_text(text_to_save, encoding="utf-8")
print(f"{log_prefix}Saved realtime text to: {output_text}")
@@ -213,6 +269,7 @@ async def run_clients_concurrent(
chunk_ms: int,
send_delay_ms: int,
delta_dump_dir: Path | None,
server_vad: bool,
num_requests: int,
concurrency: int,
) -> None:
@@ -235,6 +292,7 @@ async def run_clients_concurrent(
chunk_ms=chunk_ms,
send_delay_ms=send_delay_ms,
delta_dump_dir=per_delta_dir,
server_vad=server_vad,
request_idx=index,
total_requests=num_requests,
)
@@ -283,6 +341,11 @@ def main() -> None:
default=None,
help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
)
parser.add_argument(
"--server-vad",
action="store_true",
help="Let the server detect speech endpoints instead of sending input_audio_buffer.commit",
)
parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
parser.add_argument(
"--concurrency",
@@ -309,6 +372,7 @@ def main() -> None:
chunk_ms=args.chunk_ms,
send_delay_ms=args.send_delay_ms,
delta_dump_dir=args.delta_dump_dir,
server_vad=args.server_vad,
)
)
else:
@@ -322,6 +386,7 @@ def main() -> None:
chunk_ms=args.chunk_ms,
send_delay_ms=args.send_delay_ms,
delta_dump_dir=args.delta_dump_dir,
server_vad=args.server_vad,
num_requests=args.num_requests,
concurrency=concurrency,
)
@@ -14,6 +14,7 @@ For the full list of supported architectures across all modalities, see
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
| --- | --- | --- | --- | --- | --- |
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ |
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — |
@@ -253,16 +254,152 @@ python examples/online_serving/text_to_speech/indextts2/speech_client.py \
---
## Fish Speech S2 Pro
## Audio8 TTS Preview
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
0.6B DualAR TTS at 44.1 kHz, 11 languages, zero-shot voice cloning.
### Prerequisites
None beyond the base install: the neural audio codec is implemented in tree and
its weights (`codec.pth`) ship with the checkpoint.
### Launch
```bash
pip install fish-speech
vllm serve Audio8/Audio8-TTS-Preview-0.6b --omni --port 8092
# or:
./audio8_tts/run_server.sh
```
The deploy config auto-loads from `vllm_omni/deploy/audio8_tts.yaml` (HF
`model_type` is `arktts`). Do **not** pass `--trust-remote-code`: vllm-omni
registers its own `arktts` config, and transformers would otherwise prefer the
checkpoint's remote code and bypass it.
### Text-only synthesis
```bash
curl -X POST http://localhost:8092/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{
"input": "Welcome to Audio8 TTS.",
"response_format": "wav"
}' --output output.wav
```
### Voice cloning
```bash
curl -X POST http://localhost:8092/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{
"input": "Welcome to Audio8 TTS.",
"ref_audio": "https://example.com/reference.wav",
"ref_text": "The exact transcript of the reference recording."
}' --output cloned.wav
```
`ref_audio` also accepts a `data:audio/wav;base64,...` URL. `ref_text` is
mandatory whenever `ref_audio` is present and must match the recording.
Uploading a voice via `POST /v1/audio/voices` lets the server reuse the encoded
reference codes across requests (`voice: "<name>"`).
### Streaming
```bash
python audio8_tts/speech_client.py --text "Welcome to Audio8 TTS." --stream --output out.pcm
ffplay -f s16le -ar 44100 -ac 1 out.pcm
```
### Gradio demo
```bash
./audio8_tts/run_gradio_demo.sh # server + demo
python audio8_tts/gradio_demo.py --api-base http://localhost:8092 # demo only
```
### Benchmark
```bash
export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 # required: harness defaults to 24 kHz
vllm bench serve --omni --host 127.0.0.1 --port 8092 \
--model Audio8/Audio8-TTS-Preview-0.6b \
--backend openai-audio-speech --endpoint /v1/audio/speech \
--dataset-name seed-tts-text --dataset-path benchmarks/build_dataset/seed_tts_smoke \
--seed-tts-locale en --num-prompts 20 --num-warmups 2 \
--max-concurrency 1 --request-rate inf \
--percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun
```
Measured on 1x H20 **shared with an unrelated job holding ~80% SM**, so treat
these as a lower bound. 20 prompts from `seed_tts_smoke/en`, mean audio 4.5 s,
deploy defaults:
| | HF reference (batch=1) | vllm-omni c=1 | c=4 | c=8 |
| --- | --- | --- | --- | --- |
| RTF (mean) | 1.008 | **0.19** | 0.30 | 0.54 |
| Throughput (req/s) | 0.23 | 1.18 | 2.84 | **2.93** |
| E2E latency (mean, ms) | 4375 | 850 | 1349 | 2361 |
| Time to first packet (mean, ms) | n/a (no streaming) | 63 | 105 | 1124 |
| Underrun p99 / continuity OK | n/a | 0 s / 100% | 0 s / 100% | 0 s / 100% |
~5.3x lower per-request RTF than the upstream reference implementation at
concurrency 1, ~12.8x aggregate throughput at concurrency 8.
#### Stage-0 `max_num_seqs`: first-packet vs throughput
This subsection is operator tuning guidance, not a claim this PR optimizes
anything: the shipped deploy default is unchanged (`max_num_seqs: 4`). The
numbers below are a single run on the same contended H20 as above (one unrelated
job holding ~80% SM, no repeats), so read them as directional, not as measured
speedups. Same setup, varying stage 0's `max_num_seqs` (stage 1 stays at 1):
| stage-0 `max_num_seqs` | client concurrency | req/s | RTF | TTFP mean / p99 (ms) | underrun p99 (s) |
| --- | --- | --- | --- | --- | --- |
| 4 (default) | 1 | 1.18 | 0.19 | 63 / 67 | 0.00 |
| 4 | 4 | 2.84 | 0.30 | 105 / 148 | 0.00 |
| 4 | 8 | 2.93 | 0.54 | 1124 / 1642 | 0.00 |
| 8 | 1 | 1.16 | 0.19 | 64 / 70 | 0.00 |
| 8 | 4 | 2.88 | 0.30 | 107 / 148 | 0.00 |
| 8 | 8 | **3.72** | 0.40 | **247** / 733 | 0.22 |
| 8 | 16 | 3.91 | 0.65 | 1366 / 2530 | 0.29 |
Reading of this:
- Below the limit (`concurrency <= max_num_seqs`) the setting is irrelevant, as
expected — c=1 and c=4 are identical for both.
- At c=8, raising the limit to 8 admits every request instead of queueing half
of them: in this single run, roughly +27% throughput and ~4.5x lower
first-packet latency.
- Throughput saturates around 3.7-3.9 req/s (~17 s of audio per second). Past
c=8 you only buy queueing delay.
- The cost of admitting more streams is a 0.22-0.29 s worst-case buffer deficit
in 1 of 20 requests (`Streaming continuity OK rate` 95%), because stage 1 runs
`max_num_seqs: 1` and the codec window round-robins across streams. Client-side
prebuffering (the bundled Gradio demo holds 2 chunks) absorbs that; raising
stage 1's `max_num_seqs` is the untested knob if you need both.
The shipped default stays at 4 because every measured point there had zero
underrun. Raise it to 8 if first-packet latency under load matters more than a
rare sub-300 ms gap.
### Notes
- Output: 44.1 kHz mono; ~21.5 codec frames per second.
- No built-in speaker presets. Omit `voice` for a random timbre, or clone one.
- `max_new_tokens` caps generated codec frames (1 frame ~= 46 ms).
- Context is 2048 packed text+audio positions (`max_model_len: 2048`).
- Do not send `temperature: 0`: greedy decoding is degenerate for this
checkpoint and truncates the utterance. The deploy defaults
(0.7 / top_k 50 / top_p 0.9) mirror `generation_config.json`.
---
## Fish Speech S2 Pro
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec, which is vendored in
vLLM-Omni (no extra packages required).
### Kvcache attention fast path
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
@@ -708,13 +845,21 @@ Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and
### Streaming WebSocket
The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
The `/v1/audio/speech/stream` endpoint accepts text incrementally and, by default, synthesizes the buffered text as one continuous request on `input.done`:
```bash
python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
```
For per-sentence audio (including Indic danda `।`) before `input.done`:
```bash
python qwen3_tts/streaming_speech_client.py \
--text "नमस्ते। कैसे हो?" \
--split-granularity sentence
```
`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
```bash
@@ -0,0 +1,258 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Gradio demo for Audio8 TTS Preview via the /v1/audio/speech API.
Supports:
- text-to-speech synthesis in 11 languages
- zero-shot voice cloning from reference audio (upload, microphone, or URL)
- streaming (progressive PCM chunks) and non-streaming modes
Usage:
# Start the server first (see run_server.sh), then:
python gradio_demo.py --api-base http://localhost:8092
"""
import argparse
import base64
import io
try:
import gradio as gr
except ImportError:
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
import httpx
import numpy as np
import soundfile as sf
#: Audio8 TTS decodes at 44.1 kHz; raw PCM must be interpreted at that rate.
PCM_SAMPLE_RATE = 44100
DEFAULT_API_BASE = "http://localhost:8092"
def encode_audio_to_base64(audio_data: tuple) -> str:
"""Encode a Gradio ``(sample_rate, numpy_array)`` input as a data URL."""
sample_rate, audio_np = audio_data
if audio_np.dtype != np.int16:
if audio_np.dtype in (np.float32, np.float64):
audio_np = (np.clip(audio_np, -1.0, 1.0) * 32767).astype(np.int16)
else:
audio_np = audio_np.astype(np.int16)
buf = io.BytesIO()
sf.write(buf, audio_np, sample_rate, format="WAV")
return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}"
def build_payload(
text: str,
ref_audio: tuple | None,
ref_audio_url: str,
ref_text: str,
response_format: str = "wav",
stream: bool = False,
) -> dict:
"""Build the ``/v1/audio/speech`` payload.
Audio8 TTS has no built-in speakers: omit ``voice`` for a random timbre, or
supply ``ref_audio`` + ``ref_text`` to clone a voice.
"""
if not text or not text.strip():
raise gr.Error("Please enter text to synthesize.")
payload: dict = {
"input": text.strip(),
"response_format": "pcm" if stream else response_format,
"stream": stream,
}
if stream:
payload["stream_format"] = "audio"
ref_url = ref_audio_url.strip() if ref_audio_url else ""
if ref_audio is not None:
payload["ref_audio"] = encode_audio_to_base64(ref_audio)
elif ref_url:
payload["ref_audio"] = ref_url
if "ref_audio" in payload:
if not ref_text or not ref_text.strip():
raise gr.Error("Voice cloning requires the transcript of the reference audio.")
payload["ref_text"] = ref_text.strip()
return payload
def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format):
"""Non-streaming request; returns the whole waveform."""
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=False)
try:
with httpx.Client(timeout=300.0) as client:
resp = client.post(
f"{api_base}/v1/audio/speech",
json=payload,
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
)
except httpx.TimeoutException:
raise gr.Error("Request timed out. The server may be busy.") from None
except httpx.ConnectError:
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
if resp.status_code != 200:
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
if "application/json" in resp.headers.get("content-type", ""):
raise gr.Error(f"Server error: {resp.text}")
try:
if response_format == "pcm":
audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
return (PCM_SAMPLE_RATE, audio_np)
audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
if audio_np.ndim > 1:
audio_np = audio_np[:, 0]
return (sample_rate, audio_np.astype(np.float32))
except Exception as e:
raise gr.Error(f"Failed to decode audio: {e}") from None
def generate_speech_stream(api_base, text, ref_audio, ref_audio_url, ref_text, response_format, chunk_seconds=0.5):
"""Streaming request; yields PCM chunks for progressive playback."""
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=True)
min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
prebuffer_chunks = 2
pending: list[np.ndarray] = []
pending_len = 0
chunks_yielded = 0
prebuffer: list[np.ndarray] = []
leftover = b""
try:
with (
httpx.Client(timeout=300.0) as client,
client.stream(
"POST",
f"{api_base}/v1/audio/speech",
json=payload,
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
) as resp,
):
if resp.status_code != 200:
resp.read()
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
for chunk in resp.iter_bytes():
if not chunk:
continue
raw = leftover + chunk
# s16le: never split a sample across chunk boundaries.
usable = len(raw) - (len(raw) % 2)
leftover = raw[usable:]
if usable == 0:
continue
samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
pending.append(samples)
pending_len += len(samples)
if pending_len < min_chunk_samples:
continue
audio_chunk = np.concatenate(pending)
pending.clear()
pending_len = 0
if chunks_yielded < prebuffer_chunks:
prebuffer.append(audio_chunk)
chunks_yielded += 1
if chunks_yielded == prebuffer_chunks:
yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
prebuffer.clear()
else:
yield (PCM_SAMPLE_RATE, audio_chunk)
remaining = prebuffer + pending
if remaining:
yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
except httpx.TimeoutException:
raise gr.Error("Request timed out. The server may be busy.") from None
except httpx.ConnectError:
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
def on_stream_change(stream: bool):
"""Streaming only makes sense with raw PCM."""
if stream:
return gr.update(value="pcm", interactive=False)
return gr.update(value="wav", interactive=True)
def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
with gr.Blocks(title="Audio8 TTS Preview Demo") as demo:
gr.Markdown("# Audio8 TTS Preview 0.6B - Text to Speech")
gr.Markdown(f"**Server:** `{api_base}` | **Model:** Audio8/Audio8-TTS-Preview-0.6b | **Output:** 44.1 kHz")
with gr.Row():
with gr.Column(scale=3):
text_input = gr.Textbox(label="Text to Synthesize", placeholder="Enter text here...", lines=4)
with gr.Accordion("Voice Cloning (optional)", open=False):
gr.Markdown(
"Upload or link a short reference recording (5-30 s) and paste its **exact** transcript. "
"A transcript that does not match the audio degrades both stability and speaker similarity."
)
ref_audio = gr.Audio(label="Reference Audio", type="numpy", sources=["upload", "microphone"])
ref_audio_url = gr.Textbox(
label="Reference Audio URL (alternative to upload)",
placeholder="https://example.com/reference.wav",
lines=1,
)
ref_text = gr.Textbox(
label="Reference Audio Transcript (required for cloning)",
placeholder="Exact transcript of the reference audio...",
lines=2,
)
with gr.Row():
response_format = gr.Dropdown(
choices=["wav", "mp3", "flac", "pcm"], value="wav", label="Audio Format", scale=1
)
stream_checkbox = gr.Checkbox(
label="Stream output", value=False, info="Progressive PCM streaming", scale=1
)
generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
with gr.Column(scale=2):
audio_output = gr.Audio(label="Generated Audio", interactive=False, streaming=True, autoplay=True)
gr.Markdown(
"### About\n"
"- **Audio8 TTS Preview** — 0.6B DualAR model (slow AR + fast AR codebook decoder)\n"
"- **Languages**: Cantonese, Chinese, Dutch, English, French, German, "
"Italian, Japanese, Korean, Polish, Spanish\n"
"- **Voice cloning**: zero-shot from a reference clip + transcript\n"
"- **44.1 kHz** output, ~21.5 codec frames per second"
)
stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format]
def dispatch(stream_enabled, *args):
if stream_enabled:
yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
else:
yield generate_speech(api_base, *args)
generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
demo.queue()
return demo
def main() -> None:
parser = argparse.ArgumentParser(description="Gradio demo for Audio8 TTS Preview")
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
parser.add_argument("--host", default="0.0.0.0", help="Gradio host")
parser.add_argument("--port", type=int, default=7861, help="Gradio port")
parser.add_argument("--share", action="store_true", help="Create a public share link")
parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds of audio per yielded chunk")
args = parser.parse_args()
print(f"Connecting to vLLM server at: {args.api_base}")
build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds).launch(
server_name=args.host, server_port=args.port, share=args.share
)
if __name__ == "__main__":
main()
@@ -0,0 +1,44 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Launch the Audio8 TTS Preview server + Gradio demo together.
#
# Usage:
# ./run_gradio_demo.sh
# CUDA_VISIBLE_DEVICES=0 PORT=8092 GRADIO_PORT=7861 ./run_gradio_demo.sh
set -e
MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
PORT="${PORT:-8092}"
GRADIO_PORT="${GRADIO_PORT:-7861}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
echo "Starting Audio8 TTS Preview server (port $PORT)..."
vllm serve "$MODEL" \
--omni \
--host 0.0.0.0 \
--port "$PORT" &
SERVER_PID=$!
cleanup() {
echo "Stopping server (PID $SERVER_PID)..."
kill $SERVER_PID 2>/dev/null
wait $SERVER_PID 2>/dev/null
}
trap cleanup EXIT
echo "Waiting for server to start..."
for _ in $(seq 1 120); do
if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
echo "Server ready."
break
fi
sleep 2
done
echo "Starting Gradio demo (port $GRADIO_PORT)..."
python "$SCRIPT_DIR/gradio_demo.py" \
--api-base "http://localhost:$PORT" \
--port "$GRADIO_PORT"
@@ -0,0 +1,24 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Launch a vLLM-Omni server for Audio8 TTS Preview 0.6B.
#
# Usage:
# ./run_server.sh
# CUDA_VISIBLE_DEVICES=0 PORT=8092 ./run_server.sh
set -e
MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
PORT="${PORT:-8092}"
echo "Starting Audio8 TTS Preview server with model: $MODEL"
# --trust-remote-code is deliberately NOT passed: vllm-omni registers its own
# `arktts` config, and transformers would otherwise prefer the checkpoint's
# remote code and bypass it.
vllm serve "$MODEL" \
--omni \
--host 0.0.0.0 \
--port "$PORT"
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Client for Audio8 TTS Preview via the /v1/audio/speech endpoint.
Examples:
# Basic TTS
python speech_client.py --text "Welcome to Audio8 TTS."
# Zero-shot voice cloning (the transcript must match the reference audio)
python speech_client.py --text "Welcome to Audio8 TTS." \
--ref-audio ref.wav --ref-text "The exact transcript of the reference recording."
# Streaming PCM output
python speech_client.py --text "Welcome to Audio8 TTS." --stream --output output.pcm
"""
import argparse
import base64
import os
import httpx
DEFAULT_API_BASE = "http://localhost:8092"
DEFAULT_API_KEY = "EMPTY"
DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
#: The codec runs at 44.1 kHz, so raw PCM must be played back at that rate.
PCM_SAMPLE_RATE = 44100
def encode_audio_to_base64(audio_path: str) -> str:
"""Encode a local audio file as a base64 data URL."""
if not os.path.exists(audio_path):
raise FileNotFoundError(f"Audio file not found: {audio_path}")
ext = audio_path.lower().rsplit(".", 1)[-1]
mime_type = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}.get(
ext, "audio/wav"
)
with open(audio_path, "rb") as handle:
audio_b64 = base64.b64encode(handle.read()).decode("utf-8")
return f"data:{mime_type};base64,{audio_b64}"
def build_payload(args) -> dict:
payload = {
"model": args.model,
"input": args.text,
"response_format": args.response_format,
}
if args.ref_audio:
if args.ref_audio.startswith(("http://", "https://")):
payload["ref_audio"] = args.ref_audio
else:
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
if args.ref_text:
payload["ref_text"] = args.ref_text
if args.max_new_tokens is not None:
payload["max_new_tokens"] = args.max_new_tokens
if args.stream:
payload["stream"] = True
payload["stream_format"] = "audio"
payload["response_format"] = "pcm"
return payload
def run_tts(args) -> None:
payload = build_payload(args)
api_url = f"{args.api_base}/v1/audio/speech"
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {args.api_key}"}
print(f"Model: {args.model}")
print(f"Text: {args.text}")
if args.ref_audio:
print(f"Voice cloning: ref_audio={args.ref_audio!r} ref_text={args.ref_text!r}")
if args.stream:
output_path = args.output or "output.pcm"
with (
httpx.Client(timeout=300.0) as client,
client.stream("POST", api_url, json=payload, headers=headers) as resp,
):
if resp.status_code != 200:
print(f"Error {resp.status_code}: {resp.read().decode()}")
return
total_bytes = 0
with open(output_path, "wb") as handle:
for chunk in resp.iter_bytes():
handle.write(chunk)
total_bytes += len(chunk)
seconds = total_bytes / 2 / PCM_SAMPLE_RATE
print(f"Streamed {total_bytes} bytes (~{seconds:.2f}s of s16le @ {PCM_SAMPLE_RATE} Hz) to {output_path}")
print(f"Play with: ffplay -f s16le -ar {PCM_SAMPLE_RATE} -ac 1 {output_path}")
return
with httpx.Client(timeout=300.0) as client:
response = client.post(api_url, json=payload, headers=headers)
if response.status_code != 200:
print(f"Error {response.status_code}: {response.text}")
return
if response.headers.get("content-type", "").startswith("application/json"):
print(f"Error: {response.text}")
return
output_path = args.output or f"output.{args.response_format}"
with open(output_path, "wb") as handle:
handle.write(response.content)
print(f"Audio saved to: {output_path} ({len(response.content)} bytes)")
def main() -> None:
parser = argparse.ArgumentParser(description="Audio8 TTS Preview client")
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
parser.add_argument("--text", required=True, help="Text to synthesize")
parser.add_argument("--ref-audio", default=None, help="Reference audio for voice cloning (path or URL)")
parser.add_argument("--ref-text", default=None, help="Transcript of the reference audio")
parser.add_argument("--max-new-tokens", type=int, default=None, help="Cap the number of generated codec frames")
parser.add_argument("--stream", action="store_true", help="Stream PCM instead of returning a whole file")
parser.add_argument(
"--response-format",
default="wav",
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
help="Audio format (default: wav)",
)
parser.add_argument("--output", "-o", default=None, help="Output file path")
run_tts(parser.parse_args())
if __name__ == "__main__":
main()
@@ -24,6 +24,11 @@ Usage:
--text "Hello world. How are you? I am fine." \
--simulate-stt --stt-delay 0.1
# Opt into per-sentence synthesis (Indic danda, CJK, Latin)
python streaming_speech_client.py \
--text "नमस्ते। कैसे हो?" \
--split-granularity sentence
# Receive JSON sidecar chunks with word-level timestamps
python streaming_speech_client.py \
--text "Hello world. How are you?" \
@@ -73,8 +78,9 @@ except ImportError:
def frame_basename(msg: dict) -> str:
"""Name a file after the utterance and sentence a frame belongs to.
Every utterance is sentence 0, so the utterance index is what keeps the
files of one connection from overwriting each other.
Every utterance uses `sentence_index` for units inside that flush.
With default `split_granularity=none` that index stays 0; sentence mode
uses 0, 1, ... so files from one connection do not overwrite each other.
"""
return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
@@ -368,6 +374,13 @@ def main():
)
parser.add_argument("--speed", type=float, default=1.0, help="Playback speed (0.25-4.0)")
parser.add_argument("--max-new-tokens", type=int, default=None, help="Max tokens")
parser.add_argument("--seed", type=int, default=None, help="Sampling seed forwarded to the server")
parser.add_argument(
"--split-granularity",
default=None,
choices=["none", "sentence", "clause"],
help="Text split mode (default: server none = one request per input.done)",
)
# Base task options
parser.add_argument("--ref-audio", default=None, help="Reference audio")
@@ -409,6 +422,8 @@ def main():
"max_new_tokens",
"ref_audio",
"ref_text",
"seed",
"split_granularity",
]:
val = getattr(args, key.replace("-", "_"), None)
if val is not None: