[OmniVoice] Add two-stage TTS serving support (#2463)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
1 parent
10db95f9a9
commit
0e83ebe1d4
24 files changed
+2737
-1
No files matched your search
@@ -0,0 +1,73 @@
|
||||
# OmniVoice
|
||||
|
||||
This directory contains an offline demo for running OmniVoice TTS models with vLLM Omni. It generates speech from text and saves WAV files locally.
|
||||
|
||||
## Model Overview
|
||||
|
||||
[OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) is a zero-shot multilingual TTS model supporting 600+ languages. It uses a diffusion language model (Qwen3-0.6B backbone) with iterative masked unmasking to generate speech.
|
||||
|
||||
Three inference modes are supported:
|
||||
|
||||
- **Auto Voice**: Generate speech without any reference — the model picks a voice automatically.
|
||||
- **Voice Clone**: Clone a voice from a reference audio + transcription.
|
||||
- **Voice Design**: Control voice style via natural language instruction (e.g., "female, low pitch, british accent").
|
||||
|
||||
## Setup
|
||||
|
||||
Ensure the model is downloaded:
|
||||
|
||||
```bash
|
||||
huggingface-cli download k2-fsa/OmniVoice
|
||||
```
|
||||
|
||||
> **Note:** Voice cloning requires `transformers>=5.3.0` for `HiggsAudioV2TokenizerModel`. Auto voice and voice design modes work with `transformers>=4.57.0`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Auto voice (text only):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello, this is a test."
|
||||
```
|
||||
|
||||
Voice design (with style instruction):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "Hello, this is a test." \
|
||||
--instruct "female, low pitch, british accent"
|
||||
```
|
||||
|
||||
Voice clone (with reference audio):
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "Hello, this is a test." \
|
||||
--ref-audio ref.wav \
|
||||
--ref-text "This is the reference transcription."
|
||||
```
|
||||
|
||||
## Language Support
|
||||
|
||||
Specify a language for improved quality:
|
||||
|
||||
```bash
|
||||
python end2end.py --model k2-fsa/OmniVoice \
|
||||
--text "你好,这是一个测试。" \
|
||||
--lang zh
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
OmniVoice uses a two-stage pipeline:
|
||||
|
||||
- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text.
|
||||
- **Stage 1 (Decoder)**: HiggsAudioV2 RVQ quantizer + DAC acoustic decoder. Converts tokens to 24kHz waveform.
|
||||
|
||||
Both stages use `GPUGenerationWorker` with `OmniGenerationScheduler`.
|
||||
|
||||
## Notes
|
||||
|
||||
- Output audio is saved to `output.wav` by default. Use `--output` to change the path.
|
||||
- The model estimates duration from text automatically via `RuleDurationEstimator`.
|
||||
- Use `--stage-init-timeout` to increase the stage initialization timeout for first-time model downloads.
|
||||
@@ -0,0 +1,164 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end OmniVoice TTS inference via vLLM-Omni.
|
||||
|
||||
Supports:
|
||||
- Auto voice mode: text only → generated speech
|
||||
- Voice cloning mode: text + reference audio → cloned voice speech
|
||||
|
||||
Usage:
|
||||
# Auto voice
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello world"
|
||||
|
||||
# Voice cloning
|
||||
python end2end.py --model k2-fsa/OmniVoice --text "Hello" \
|
||||
--ref-audio ref.wav --ref-text "reference transcription"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
def run_e2e():
|
||||
parser = argparse.ArgumentParser(description="OmniVoice E2E TTS inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="k2-fsa/OmniVoice",
|
||||
help="Model name or path (HuggingFace or local)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-config",
|
||||
type=str,
|
||||
default="vllm_omni/model_executor/stage_configs/omnivoice.yaml",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Hello, this is a test of the OmniVoice text to speech system.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Reference audio for voice cloning (WAV file)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Transcription of reference audio",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lang",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Language code (e.g., 'en', 'zh')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instruct",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Voice design instruction (e.g., 'female, low pitch, british accent')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="output.wav",
|
||||
help="Output audio file path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-init-timeout",
|
||||
type=int,
|
||||
default=600,
|
||||
help="Stage initialization timeout in seconds",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.stage_config):
|
||||
raise FileNotFoundError(f"Stage config not found: {args.stage_config}")
|
||||
|
||||
print(f"Initializing OmniVoice with model={args.model}")
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_config,
|
||||
trust_remote_code=True,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
print("Model initialized. Preparing inputs...")
|
||||
|
||||
# Build prompt
|
||||
mm_processor_kwargs = {}
|
||||
multi_modal_data = {}
|
||||
|
||||
if args.ref_audio:
|
||||
if not os.path.exists(args.ref_audio):
|
||||
raise FileNotFoundError(f"Reference audio not found: {args.ref_audio}")
|
||||
|
||||
import librosa
|
||||
|
||||
audio_signal, sr = librosa.load(args.ref_audio, sr=None)
|
||||
multi_modal_data["audio"] = (audio_signal.astype(np.float32), sr)
|
||||
mm_processor_kwargs["ref_text"] = args.ref_text or ""
|
||||
mm_processor_kwargs["sample_rate"] = sr
|
||||
|
||||
if args.lang:
|
||||
mm_processor_kwargs["lang"] = args.lang
|
||||
if args.instruct:
|
||||
mm_processor_kwargs["instruct"] = args.instruct
|
||||
|
||||
prompts = {"prompt": args.text}
|
||||
if multi_modal_data:
|
||||
prompts["multi_modal_data"] = multi_modal_data
|
||||
if mm_processor_kwargs:
|
||||
prompts["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
|
||||
print(f"Generating speech for: {args.text}")
|
||||
|
||||
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list))
|
||||
|
||||
print(f"Received {len(outputs)} outputs.")
|
||||
for i, output in enumerate(outputs):
|
||||
try:
|
||||
ro = output.request_output
|
||||
if ro is None:
|
||||
print("No request_output found.")
|
||||
continue
|
||||
|
||||
mm = getattr(ro, "multimodal_output", None)
|
||||
if not mm and ro.outputs:
|
||||
mm = getattr(ro.outputs[0], "multimodal_output", None)
|
||||
|
||||
if mm:
|
||||
print(f"Multimodal output keys: {mm.keys()}")
|
||||
if "audio" in mm:
|
||||
audio_out = mm["audio"]
|
||||
sr = mm.get("sr", 24000)
|
||||
if isinstance(audio_out, np.ndarray):
|
||||
audio_np = audio_out
|
||||
else:
|
||||
audio_np = audio_out.cpu().numpy().squeeze()
|
||||
out_path = args.output if i == 0 else f"output_{i}.wav"
|
||||
sf.write(out_path, audio_np, sr)
|
||||
print(f"Saved audio to {out_path} ({sr}Hz, {len(audio_np) / sr:.2f}s)")
|
||||
else:
|
||||
print("No multimodal output found.")
|
||||
except Exception as e:
|
||||
print(f"Error inspecting output: {e}")
|
||||
|
||||
omni.close()
|
||||
print("Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_e2e()
|
||||
Reference in new issue
Block a user