[Model] Add Breeze-TTS-2 two-stage AR TTS support (talker + codec, streaming) (#7084)

Signed-off-by: noname <liuzhiwei08@kuaishou.com>
Signed-off-by: noname <133508552+lzwnoname@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: noname <liuzhiwei08@kuaishou.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-14 09:55:53 -07:00
1 parent e0c4f97f32
commit ea1a416a17
46 files changed
+6067 -4

No files matched your search

@@ -16,6 +16,7 @@ list of supported architectures across all modalities, see
|---|---|---|---|---|---|---|
| AuK | `tencent/AuK`, `tencent/AuK-Flash` (assembled with `tools/prepare_auk_checkpoint.py`) | 2 (thinker encoder + DiT) | ✓ | — | instruction-driven editing, enhancement, separation; no example script (use `vllm_omni.model_extras.auk`) | 24 kHz |
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
| Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz |
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz |
@@ -662,6 +663,37 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
### Notes
- `--num-prompts N` replicates the prompt for performance measurement.
- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
- Run `--help` for the full argument surface.
---
## Breeze-TTS-2
Two-stage AR TTS (T5Gemma2 + Qwen3 talker with a depth decoder → bundled Qwen3-TTS codec) at 24 kHz mono. The prompt builder picks one of four templates automatically: plain, voice design (`--instruction`), clone (`--ref-audio` + `--ref-text`), or voice direction (reference + `--instruction`).
### Quick start
```bash
python examples/offline_inference/text_to_speech/breeze_tts_2/end2end.py \
--model BreezeBlue/Breeze-TTS-2 \
--text "Hello, this is Breeze TTS 2 running on vLLM Omni."
```
### Voice direction
```bash
python examples/offline_inference/text_to_speech/breeze_tts_2/end2end.py \
--text "We need to discuss what happened last night." \
--ref-audio /path/to/reference.wav \
--ref-text "The exact transcript of the reference audio." \
--instruction "Speak slowly with a restrained, serious tone."
```
### Notes
- Output: 24 kHz mono WAV; `--max-new-tokens` caps generated frames (80 ms each).
- Speaker tags `--voice S0`..`S9` (default `S0`) select the timbre in non-reference modes; with `--ref-audio` the timbre comes from the reference clip.
- Deploy config: `vllm_omni/deploy/breeze_tts_2.yaml` (auto-loaded by HF `model_type`); see [`recipes/BreezeBlue/Breeze-TTS-2.md`](../../../recipes/BreezeBlue/Breeze-TTS-2.md) for the online-serving recipe.
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Offline Breeze-TTS-2 inference example (two-stage talker + codec pipeline).
Uses the async-chunk deploy config (vllm_omni/deploy/breeze_tts_2.yaml).
Reference checkpoint: BreezeBlue/Breeze-TTS-2.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
import soundfile as sf
import torch
from transformers import AutoConfig
from vllm_omni import Omni
from vllm_omni.model_executor.models.breeze_tts_2.prompt_builder import (
BreezeTTS2PromptBuilder,
)
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
REPO_ROOT = Path(__file__).resolve().parents[4]
SAMPLE_RATE = 24_000
def parse_args():
parser = TrackingArgumentParser(description="Offline Breeze-TTS-2 inference")
parser.add_argument(
"--model",
type=str,
default="BreezeBlue/Breeze-TTS-2",
help="Breeze-TTS-2 model path or HuggingFace repo ID.",
)
parser.add_argument(
"--text",
type=str,
default="Hello, this is Breeze TTS 2 running on vLLM Omni.",
help="Text to synthesize.",
)
parser.add_argument(
"--voice",
type=str,
default="S0",
help="Speaker tag such as S0..S9.",
)
parser.add_argument(
"--instruction",
type=str,
default=None,
help="Optional style instruction (voice design / voice direction).",
)
parser.add_argument(
"--ref-audio",
type=str,
default=None,
help="Optional reference wav path for cloning (requires --ref-text).",
)
parser.add_argument(
"--ref-text",
type=str,
default=None,
help="Exact transcript of the reference audio.",
)
parser.add_argument(
"--max-new-tokens",
type=int,
default=None,
help="Cap on generated codec frames (one frame is 80 ms of audio).",
)
parser.add_argument(
"--output-dir",
type=str,
default="output_audio",
help="Directory for output WAV files.",
)
parser.add_argument(
"--deploy-config",
type=str,
default=None,
help="Override the deploy config path. If unset, auto-loads "
"vllm_omni/deploy/breeze_tts_2.yaml based on the HF model_type.",
)
return parser.parse_args()
def resolve_template(args) -> str:
if args.ref_audio is not None:
return "ref_edit_tata" if args.instruction else "ref_clone_tata"
return "tts_instruction" if args.instruction else "tts_plain"
def reference_payload(ref_audio_path: str | None) -> dict:
"""Load the reference clip into the payload keys the prompt builder reads.
``BreezeTTS2PromptBuilder`` consumes ``ref_audio`` (a waveform) plus
``ref_audio_sample_rate``; a bare path field is not read.
"""
if ref_audio_path is None:
return {}
waveform, sample_rate = sf.read(ref_audio_path, dtype="float32", always_2d=True)
# Down-mix to the single 1-D clip the reference encoder expects.
waveform = waveform.mean(axis=1)
return {"ref_audio": waveform, "ref_audio_sample_rate": int(sample_rate)}
def build_prompt(args) -> dict:
config = AutoConfig.from_pretrained(args.model, trust_remote_code=True)
reference_encoder = None
if args.ref_audio is not None:
from vllm_omni.model_executor.models.breeze_tts_2.audio_tokenizer import (
BreezeReferenceAudioTokenizer,
)
reference_encoder = BreezeReferenceAudioTokenizer.from_pretrained(
args.model,
num_codebooks=int(getattr(config, "num_codebooks", 16)),
device_map="cpu",
)
builder = BreezeTTS2PromptBuilder.from_pretrained(
args.model,
config,
reference_audio_encoder=reference_encoder,
)
payload = {
"text": args.text,
"speaker": args.voice,
**reference_payload(args.ref_audio),
}
if args.instruction:
payload["instruction"] = args.instruction
if args.ref_text:
payload["ref_text"] = args.ref_text
prompt = builder.build(payload, template=resolve_template(args))
if args.max_new_tokens is not None:
prompt["additional_information"]["breeze_max_new_frames"] = int(args.max_new_tokens)
return prompt
def extract_audio(multimodal_output: dict) -> torch.Tensor:
"""Extract the final complete audio tensor from multimodal output.
The output processor concatenates per-step delta tensors under
``model_outputs``. Falls back to ``audio`` for backwards compat.
"""
audio = multimodal_output.get("model_outputs")
if audio is None:
audio = multimodal_output.get("audio")
if audio is None:
raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
if isinstance(audio, list):
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
if not valid:
raise ValueError("Audio list is empty or all elements are None.")
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
return torch.as_tensor(audio).float().cpu().reshape(-1)
def main():
args = parse_args()
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
engine = Omni(
model=args.model,
deploy_config=args.deploy_config,
)
prompt = build_prompt(args)
print(f"Model : {args.model}")
print(f"Text : {args.text}")
print(f"Template : {resolve_template(args)}")
print(f"Output dir : {output_dir}")
print(f"Prompt len : {len(prompt['prompt_token_ids'])} tokens")
t_start = time.perf_counter()
outputs = engine.generate([prompt])
elapsed = time.perf_counter() - t_start
request_output = outputs[0]
mm = request_output.outputs[0].multimodal_output
audio = extract_audio(mm)
duration = audio.numel() / SAMPLE_RATE
rtf = elapsed / duration if duration > 0 else float("inf")
output_path = output_dir / "output.wav"
sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
print(f"Saved : {output_path}")
print(f"Duration : {duration:.2f}s")
print(f"Inference : {elapsed:.2f}s")
print(f"RTF : {rtf:.3f}")
if __name__ == "__main__":
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
main()