[Model] Add Breeze-TTS-2 two-stage AR TTS support (talker + codec, streaming) (#7084)

Signed-off-by: noname <liuzhiwei08@kuaishou.com>
Signed-off-by: noname <133508552+lzwnoname@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: noname <liuzhiwei08@kuaishou.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-14 09:55:53 -07:00
1 parent e0c4f97f32
commit ea1a416a17
46 files changed
+6067 -4

No files matched your search

+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Compare upstream and vLLM-Omni Breeze-TTS-2 golden artifacts."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import soundfile as sf
import torch
TEMPLATES = {
"plain": "tts_plain",
"instruction": "tts_instruction",
"reference_edit": "ref_edit_tata",
}
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _audio_metrics(path: Path) -> dict[str, object]:
data, sample_rate = sf.read(path, dtype="float32")
return {
"path": str(path),
"bytes": path.stat().st_size,
"sha256": _sha256(path),
"samples": int(data.shape[0]),
"sample_rate": int(sample_rate),
"channels": int(1 if data.ndim == 1 else data.shape[1]),
"duration_s": round(float(data.shape[0]) / sample_rate, 6),
"finite": bool(torch.isfinite(torch.from_numpy(data)).all().item()),
"peak": round(float(abs(data).max()), 6),
"rms": round(float((data**2).mean() ** 0.5), 6),
}
def _vector_metrics(upstream: torch.Tensor, actual: torch.Tensor) -> dict[str, float]:
difference = upstream.double() - actual.double()
return {
"mae": round(float(difference.abs().mean()), 8),
"rmse": round(float(difference.pow(2).mean().sqrt()), 8),
"max_abs_error": round(float(difference.abs().max()), 8),
"relative_l2": round(float(difference.norm() / upstream.norm()), 8),
"cosine": round(
float(torch.dot(upstream.double(), actual.double()) / (upstream.norm() * actual.norm())),
8,
),
}
def _load_vllm_payload(vllm_dir: Path, template: str) -> dict[str, object]:
matches = []
for path in vllm_dir.glob("breeze_*.pt"):
payload = torch.load(path, weights_only=False)
if payload.get("template") == template:
matches.append(payload)
if len(matches) != 1:
raise ValueError(f"expected one vLLM dump for {template!r}, found {len(matches)}")
return matches[0]
def _longest_exact_frame_prefix(upstream: torch.Tensor, actual: torch.Tensor) -> int:
count = min(int(upstream.shape[0]), int(actual.shape[0]))
equal = (upstream[:count] == actual[:count]).all(dim=-1)
prefix = 0
for is_equal in equal.tolist():
if not is_equal:
break
prefix += 1
return prefix
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--upstream-dir", type=Path, required=True)
parser.add_argument("--vllm-dir", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--plain-audio", type=Path)
parser.add_argument("--instruction-audio", type=Path)
parser.add_argument("--reference-audio", type=Path)
args = parser.parse_args()
audio_paths = {
"plain": args.plain_audio,
"instruction": args.instruction_audio,
"reference_edit": args.reference_audio,
}
report: dict[str, object] = {
"upstream_dir": str(args.upstream_dir.resolve()),
"vllm_dir": str(args.vllm_dir.resolve()),
"cases": [],
}
for name, template in TEMPLATES.items():
upstream_codes = torch.load(args.upstream_dir / name / "codes.pt", weights_only=False)
upstream_prompt = torch.load(args.upstream_dir / name / "prompt_ids.pt", weights_only=False)
actual_payload = _load_vllm_payload(args.vllm_dir, template)
actual_codes = actual_payload["codes"]
actual_prompt = actual_payload["prompt_ids"]
common_frames = min(int(upstream_codes.shape[0]), int(actual_codes.shape[0]))
code_equal = upstream_codes[:common_frames] == actual_codes[:common_frames]
frame_equal = code_equal.all(dim=-1)
prefill_metrics = None
upstream_hidden_path = args.upstream_dir / name / "prefill_hidden.pt"
if upstream_hidden_path.exists():
prefill_files = [
path
for path in args.vllm_dir.glob("prefill_*.pt")
if torch.load(path, weights_only=False).get("template") == template
]
if len(prefill_files) == 1:
actual_hidden = torch.load(prefill_files[0], weights_only=False)["hidden"]
prefill_metrics = _vector_metrics(
torch.load(upstream_hidden_path, weights_only=False),
actual_hidden,
)
entry: dict[str, object] = {
"name": name,
"template": template,
"prompt": {
"upstream_tokens": int(upstream_prompt.numel()),
"vllm_tokens": int(actual_prompt.numel()),
"exact": bool(torch.equal(upstream_prompt, actual_prompt)),
},
"frames": {
"upstream": int(upstream_codes.shape[0]),
"vllm": int(actual_codes.shape[0]),
"codebooks": int(upstream_codes.shape[-1]),
"exact_sequence": bool(torch.equal(upstream_codes, actual_codes)),
"common_frames": common_frames,
"exact_common_frames": int(frame_equal.sum()),
"longest_exact_prefix": _longest_exact_frame_prefix(upstream_codes, actual_codes),
"exact_codes": int(code_equal.sum()),
"total_common_codes": int(code_equal.numel()),
"code_accuracy": round(float(code_equal.float().mean()), 8),
"codebook0_accuracy": round(
float((upstream_codes[:common_frames, 0] == actual_codes[:common_frames, 0]).float().mean()),
8,
),
"range_valid": bool(
upstream_codes.min() >= 0
and upstream_codes.max() < 2048
and actual_codes.min() >= 0
and actual_codes.max() < 2048
),
},
"prefill_hidden": prefill_metrics,
}
if audio_paths[name] is not None and Path(audio_paths[name]).exists(): # type: ignore[arg-type]
entry["audio"] = _audio_metrics(Path(audio_paths[name])) # type: ignore[arg-type]
report["cases"].append(entry)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()
+200
View File
@@ -0,0 +1,200 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Generate deterministic Breeze-TTS-2 golden frames with the upstream runtime.
This tool is intentionally separate from vLLM-Omni: it imports the unmodified
``breezeblue-ai/breeze-tts`` reference implementation, fixes greedy decoding,
and writes both generated codec frames and decoded audio for offline parity
checks. It is a diagnostic tool, not a serving path.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import soundfile as sf
import torch
def _plain_template(upstream_templates):
"""The upstream CLI does not register tts_plain; reuse its segment builder."""
return SimpleNamespace(
name="tts_plain",
required_fields=("text",),
build_segments=upstream_templates._tts_plain_segments,
build_negative_segments=None,
build_dual_branches=None,
)
def _greedy(model) -> None:
model.generation_config.do_sample = False
model.generation_config.temperature = None
model.generation_config.top_p = None
model.generation_config.top_k = None
depth = model.depth_decoder.generation_config
depth.depth_decoder_do_sample = False
depth.do_sample = False
depth.temperature = None
depth.top_p = None
depth.top_k = None
def _valid_frames(sequences: torch.Tensor, pad_token_id: int) -> torch.Tensor:
frames = sequences[0].detach().cpu().to(torch.long)
if frames.ndim == 1:
frames = frames.unsqueeze(0)
is_pad = (frames == pad_token_id).all(dim=-1)
if bool(is_pad.any()):
frames = frames[: int(torch.argmax(is_pad.int()).item())]
return frames.contiguous()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--upstream", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--reference-audio", type=Path)
parser.add_argument("--max-new-tokens", type=int, default=32)
args = parser.parse_args()
args.output_dir.mkdir(parents=True, exist_ok=True)
upstream_posix = str(args.upstream.resolve())
if upstream_posix not in sys.path:
sys.path.insert(0, upstream_posix)
from breeze_infer.runtime import load_runtime, resolve_device
from breeze_infer.templates import get_template, prepare_inputs
device = resolve_device()
# ``torch.manual_seed`` seeds CUDA RNGs as well, so no device-specific
# seeding call is needed here.
torch.manual_seed(0)
tokenizer, model, audio_tokenizer = load_runtime(
args.model,
device=device,
attn_implementation="eager",
)
# Do not enable Transformers 4.x's ``fix_mistral_regex`` compatibility
# path: vLLM-Omni runs with Transformers 5.x, and direct tokenization checks
# showed both runtimes produce the same token IDs without that patch.
_greedy(model)
model.eval()
first_backbone_hidden: list[torch.Tensor] = []
def _capture_first_backbone_hidden(_module, _inputs, output):
if first_backbone_hidden:
return
hidden = getattr(output, "last_hidden_state", None)
if isinstance(hidden, torch.Tensor):
first_backbone_hidden.append(hidden[0, -1].detach().cpu().to(torch.float32).contiguous())
model.backbone_model.register_forward_hook(_capture_first_backbone_hidden)
cases = [
{
"name": "plain",
"template": _plain_template(sys.modules["breeze_infer.templates"]),
"request": {
"id": "golden-plain",
"text": "Golden plain alignment check.",
"speaker": "S0",
},
},
{
"name": "instruction",
"template": get_template("tts_instruction"),
"request": {
"id": "golden-instruction",
"text": "Golden instruction alignment check.",
"instruction": "Speak calmly, warmly, and clearly.",
"speaker": "S0",
},
},
]
if args.reference_audio is not None:
cases.append(
{
"name": "reference_edit",
"template": get_template("ref_edit_tata"),
"request": {
"id": "golden-reference-edit",
"text": "Golden reference alignment check.",
"instruction": "Speak calmly and clearly.",
"ref_audio_path": str(args.reference_audio),
"ref_text": "Hello from Breeze.",
"speaker": "S0",
},
}
)
manifest = {
"runtime": "upstream breezeblue-ai/breeze-tts",
"model": str(args.model.resolve()),
"upstream": upstream_posix,
"device": device,
"seed": 0,
"max_new_tokens": args.max_new_tokens,
"decoding": "greedy",
"cfg_scale": 1.0,
"cases": [],
}
for case in cases:
first_backbone_hidden.clear()
inputs = prepare_inputs(
tokenizer,
audio_tokenizer,
model,
[case["request"]],
case["template"],
guidance_scale=1.0,
guidance_scale_ref=None,
guidance_scale_ins=None,
)
output = model.generate(
**inputs,
max_new_tokens=args.max_new_tokens,
do_sample=False,
return_dict_in_generate=True,
output_audio=True,
audio_tokenizer=audio_tokenizer,
)
frames = _valid_frames(output.sequences, model.config.codebook_pad_token_id)
audio = output.audio[0].detach().cpu().to(torch.float32).reshape(-1)
case_dir = args.output_dir / case["name"]
case_dir.mkdir(parents=True, exist_ok=True)
torch.save(frames, case_dir / "codes.pt")
prompt_ids = inputs["input_ids"].detach().cpu().to(torch.long).reshape(-1)
torch.save(prompt_ids, case_dir / "prompt_ids.pt")
if first_backbone_hidden:
torch.save(first_backbone_hidden[0], case_dir / "prefill_hidden.pt")
sf.write(case_dir / "upstream.wav", audio.numpy(), 24_000, subtype="PCM_16")
entry = {
"name": case["name"],
"request": case["request"],
"prompt_tokens": int(prompt_ids.numel()),
"frames": int(frames.shape[0]),
"codebooks": int(frames.shape[-1]) if frames.ndim == 2 else 0,
"min_code": int(frames.min().item()) if frames.numel() else None,
"max_code": int(frames.max().item()) if frames.numel() else None,
"audio_samples": int(audio.numel()),
"audio_sha256": __import__("hashlib").sha256((case_dir / "upstream.wav").read_bytes()).hexdigest(),
}
manifest["cases"].append(entry)
print(json.dumps(entry), flush=True)
(args.output_dir / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
if __name__ == "__main__":
main()