[OmniVoice] Add two-stage TTS serving support (#2463)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed
1 parent
10db95f9a9
commit
0e83ebe1d4
24 files changed
+2737
-1
No files matched your search
@@ -0,0 +1,73 @@
|
||||
# OmniVoice
|
||||
|
||||
This directory contains an offline demo for running OmniVoice TTS models with vLLM Omni. It generates speech from text and saves WAV files locally.
|
||||
|
||||
## Model Overview
|
||||
|
||||
[OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) is a zero-shot multilingual TTS model supporting 600+ languages. It uses a diffusion language model (Qwen3-0.6B backbone) with iterative masked unmasking to generate speech.
|
||||
|
||||
Three inference modes are supported:
|
||||
|
||||
- **Auto Voice**: Generate speech without any reference — the model picks a voice automatically.
|
||||
- **Voice Clone**: Clone a voice from a reference audio + transcription.
|
||||
- **Voice Design**: Control voice style via natural language instruction (e.g., "female, low pitch, british accent").
|
||||
|
||||
## Setup
|
||||
|
||||
Ensure the model is downloaded:
|
||||
|
||||
```bash
|
||||
huggingface-cli download k2-fsa/OmniVoice
|
||||
```
|
||||
|
||||
> **Note:** Voice cloning requires `transformers>=5.3.0` for `HiggsAudioV2TokenizerModel`. Auto voice and voice design modes work with `transformers>=4.57.0`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Auto voice (text only):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello, this is a test."
|
||||
```
|
||||
|
||||
Voice design (with style instruction):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "Hello, this is a test." \
|
||||
--instruct "female, low pitch, british accent"
|
||||
```
|
||||
|
||||
Voice clone (with reference audio):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "Hello, this is a test." \
|
||||
--ref-audio ref.wav \
|
||||
--ref-text "This is the reference transcription."
|
||||
```
|
||||
|
||||
## Language Support
|
||||
|
||||
Specify a language for improved quality:
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "你好，这是一个测试。" \
|
||||
--lang zh
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
OmniVoice uses a two-stage pipeline:
|
||||
|
||||
- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text.
|
||||
- **Stage 1 (Decoder)**: HiggsAudioV2 RVQ quantizer + DAC acoustic decoder. Converts tokens to 24kHz waveform.
|
||||
|
||||
Both stages use `GPUGenerationWorker` with `OmniGenerationScheduler`.
|
||||
|
||||
## Notes
|
||||
|
||||
- Output audio is saved to `output.wav` by default. Use `--output` to change the path.
|
||||
- The model estimates duration from text automatically via `RuleDurationEstimator`.
|
||||
- Use `--stage-init-timeout` to increase the stage initialization timeout for first-time model downloads.
|
||||
@@ -0,0 +1,164 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end OmniVoice TTS inference via vLLM-Omni.
|
||||
|
||||
Supports:
|
||||
- Auto voice mode: text only → generated speech
|
||||
- Voice cloning mode: text + reference audio → cloned voice speech
|
||||
|
||||
Usage:
|
||||
# Auto voice
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello world"
|
||||
|
||||
# Voice cloning
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello" \
|
||||
--ref-audio ref.wav --ref-text "reference transcription"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
def run_e2e():
|
||||
parser = argparse.ArgumentParser(description="OmniVoice E2E TTS inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="k2-fsa/OmniVoice",
|
||||
help="Model name or path (HuggingFace or local)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-config",
|
||||
type=str,
|
||||
default="vllm_omni/model_executor/stage_configs/omnivoice.yaml",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Hello, this is a test of the OmniVoice text to speech system.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Reference audio for voice cloning (WAV file)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Transcription of reference audio",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lang",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Language code (e.g., 'en', 'zh')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instruct",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Voice design instruction (e.g., 'female, low pitch, british accent')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="output.wav",
|
||||
help="Output audio file path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-init-timeout",
|
||||
type=int,
|
||||
default=600,
|
||||
help="Stage initialization timeout in seconds",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.stage_config):
|
||||
raise FileNotFoundError(f"Stage config not found: {args.stage_config}")
|
||||
|
||||
print(f"Initializing OmniVoice with model={args.model}")
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_config,
|
||||
trust_remote_code=True,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
print("Model initialized. Preparing inputs...")
|
||||
|
||||
# Build prompt
|
||||
mm_processor_kwargs = {}
|
||||
multi_modal_data = {}
|
||||
|
||||
if args.ref_audio:
|
||||
if not os.path.exists(args.ref_audio):
|
||||
raise FileNotFoundError(f"Reference audio not found: {args.ref_audio}")
|
||||
|
||||
import librosa
|
||||
|
||||
audio_signal, sr = librosa.load(args.ref_audio, sr=None)
|
||||
multi_modal_data["audio"] = (audio_signal.astype(np.float32), sr)
|
||||
mm_processor_kwargs["ref_text"] = args.ref_text or ""
|
||||
mm_processor_kwargs["sample_rate"] = sr
|
||||
|
||||
if args.lang:
|
||||
mm_processor_kwargs["lang"] = args.lang
|
||||
if args.instruct:
|
||||
mm_processor_kwargs["instruct"] = args.instruct
|
||||
|
||||
prompts = {"prompt": args.text}
|
||||
if multi_modal_data:
|
||||
prompts["multi_modal_data"] = multi_modal_data
|
||||
if mm_processor_kwargs:
|
||||
prompts["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
|
||||
print(f"Generating speech for: {args.text}")
|
||||
|
||||
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list))
|
||||
|
||||
print(f"Received {len(outputs)} outputs.")
|
||||
for i, output in enumerate(outputs):
|
||||
try:
|
||||
ro = output.request_output
|
||||
if ro is None:
|
||||
print("No request_output found.")
|
||||
continue
|
||||
|
||||
mm = getattr(ro, "multimodal_output", None)
|
||||
if not mm and ro.outputs:
|
||||
mm = getattr(ro.outputs[0], "multimodal_output", None)
|
||||
|
||||
if mm:
|
||||
print(f"Multimodal output keys: {mm.keys()}")
|
||||
if "audio" in mm:
|
||||
audio_out = mm["audio"]
|
||||
sr = mm.get("sr", 24000)
|
||||
if isinstance(audio_out, np.ndarray):
|
||||
audio_np = audio_out
|
||||
else:
|
||||
audio_np = audio_out.cpu().numpy().squeeze()
|
||||
out_path = args.output if i == 0 else f"output_{i}.wav"
|
||||
sf.write(out_path, audio_np, sr)
|
||||
print(f"Saved audio to {out_path} ({sr}Hz, {len(audio_np) / sr:.2f}s)")
|
||||
else:
|
||||
print("No multimodal output found.")
|
||||
except Exception as e:
|
||||
print(f"Error inspecting output: {e}")
|
||||
|
||||
omni.close()
|
||||
print("Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_e2e()
|
||||
@@ -0,0 +1,131 @@
|
||||
# OmniVoice
|
||||
|
||||
## Model Overview
|
||||
|
||||
| Model | Description |
|
||||
|-------|-------------|
|
||||
| `k2-fsa/OmniVoice` | Zero-shot multilingual TTS (600+ languages) with diffusion language model (Qwen3-0.6B backbone) |
|
||||
|
||||
> **Note:** Requires `transformers>=5.3.0` for voice cloning (HiggsAudioV2 tokenizer). Auto voice and voice design work with `transformers>=4.57.0`.
|
||||
|
||||
## Launch the Server
|
||||
|
||||
```bash
|
||||
vllm serve k2-fsa/OmniVoice \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--trust-remote-code
|
||||
```
|
||||
|
||||
Or use the convenience script:
|
||||
|
||||
```bash
|
||||
./run_server.sh
|
||||
```
|
||||
|
||||
## Send TTS Request
|
||||
|
||||
### Using curl
|
||||
|
||||
```bash
|
||||
# Basic TTS (auto voice)
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
### Using Python
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
response = httpx.post(
|
||||
"http://localhost:8091/v1/audio/speech",
|
||||
json={
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"response_format": "wav",
|
||||
},
|
||||
timeout=300.0,
|
||||
)
|
||||
|
||||
with open("output.wav", "wb") as f:
|
||||
f.write(response.content)
|
||||
```
|
||||
|
||||
### Using OpenAI SDK
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
|
||||
|
||||
response = client.audio.speech.create(
|
||||
model="k2-fsa/OmniVoice",
|
||||
voice="default",
|
||||
input="Hello, how are you?",
|
||||
)
|
||||
|
||||
response.stream_to_file("output.wav")
|
||||
```
|
||||
|
||||
### Using the CLI Client
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/omnivoice
|
||||
|
||||
# Basic TTS
|
||||
python speech_client.py --text "Hello, how are you?"
|
||||
|
||||
# Specify language for improved quality
|
||||
python speech_client.py --text "Bonjour, comment allez-vous?" --language French
|
||||
```
|
||||
|
||||
The CLI client supports:
|
||||
|
||||
- `--api-base`: API base URL (default: `http://localhost:8091`)
|
||||
- `--model` (or `-m`): Model name (default: `k2-fsa/OmniVoice`)
|
||||
- `--text`: Text to synthesize (required)
|
||||
- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
|
||||
- `--language`: Language hint (default: Auto)
|
||||
- `--output` (or `-o`): Output file path (default: `omnivoice_output.wav`)
|
||||
|
||||
## Inference Modes
|
||||
|
||||
OmniVoice supports three inference modes. Currently, **auto voice** is supported through the online Speech API. Voice cloning and voice design are available via offline inference (see `examples/offline_inference/omnivoice/`).
|
||||
|
||||
| Mode | Description | Online API | Offline |
|
||||
|------|-------------|:----------:|:-------:|
|
||||
| Auto Voice | Generate speech without reference | Yes | Yes |
|
||||
| Voice Clone | Clone from reference audio + transcript | - | Yes |
|
||||
| Voice Design | Control style via natural language instruction | - | Yes |
|
||||
|
||||
## Architecture
|
||||
|
||||
OmniVoice uses a single-stage diffusion pipeline:
|
||||
|
||||
- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative masked unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text, then decodes to 24kHz waveform via HiggsAudioV2 RVQ quantizer + DAC acoustic decoder.
|
||||
|
||||
## API Parameters
|
||||
|
||||
OmniVoice uses the standard `/v1/audio/speech` endpoint. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full documentation.
|
||||
|
||||
Key parameters:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `input` | string | **required** | Text to synthesize |
|
||||
| `voice` | string | "default" | Voice name |
|
||||
| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
|
||||
| `speed` | float | 1.0 | Playback speed (0.25-4.0) |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
1. **TTS model did not produce audio output**: Ensure the model is fully downloaded (`huggingface-cli download k2-fsa/OmniVoice`)
|
||||
2. **Connection refused**: Make sure the server is running on the correct port
|
||||
3. **Out of memory**: Reduce `--gpu-memory-utilization` (default stage config uses 0.5)
|
||||
4. **Slow first request**: The model performs warmup on first inference; subsequent requests are faster
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for OmniVoice TTS
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-k2-fsa/OmniVoice}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
echo "Starting OmniVoice server with model: $MODEL"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--omni
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Client for OmniVoice TTS via /v1/audio/speech endpoint.
|
||||
|
||||
Examples:
|
||||
# Basic TTS (auto voice)
|
||||
python speech_client.py --text "Hello, how are you?"
|
||||
|
||||
# Specify language
|
||||
python speech_client.py --text "Bonjour, comment allez-vous?" --language French
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
"""Generate speech via /v1/audio/speech API."""
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"voice": "default",
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
|
||||
if args.language:
|
||||
payload["language"] = args.language
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
if args.language:
|
||||
print(f"Language: {args.language}")
|
||||
print("Generating audio...")
|
||||
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
output_path = args.output or "omnivoice_output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="OmniVoice TTS client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default="k2-fsa/OmniVoice", help="Model name")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--language", default=None, help="Language hint (e.g., English, Chinese, French)")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio format (default: wav)",
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user