[Model] Add Breeze-TTS-2 two-stage AR TTS support (talker + codec, streaming) (#7084)
Signed-off-by: noname <liuzhiwei08@kuaishou.com> Signed-off-by: noname <133508552+lzwnoname@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: noname <liuzhiwei08@kuaishou.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
46 files changed
+6067
-4
No files matched your search
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Compare upstream and vLLM-Omni Breeze-TTS-2 golden artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
TEMPLATES = {
|
||||
"plain": "tts_plain",
|
||||
"instruction": "tts_instruction",
|
||||
"reference_edit": "ref_edit_tata",
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _audio_metrics(path: Path) -> dict[str, object]:
|
||||
data, sample_rate = sf.read(path, dtype="float32")
|
||||
return {
|
||||
"path": str(path),
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
"samples": int(data.shape[0]),
|
||||
"sample_rate": int(sample_rate),
|
||||
"channels": int(1 if data.ndim == 1 else data.shape[1]),
|
||||
"duration_s": round(float(data.shape[0]) / sample_rate, 6),
|
||||
"finite": bool(torch.isfinite(torch.from_numpy(data)).all().item()),
|
||||
"peak": round(float(abs(data).max()), 6),
|
||||
"rms": round(float((data**2).mean() ** 0.5), 6),
|
||||
}
|
||||
|
||||
|
||||
def _vector_metrics(upstream: torch.Tensor, actual: torch.Tensor) -> dict[str, float]:
|
||||
difference = upstream.double() - actual.double()
|
||||
return {
|
||||
"mae": round(float(difference.abs().mean()), 8),
|
||||
"rmse": round(float(difference.pow(2).mean().sqrt()), 8),
|
||||
"max_abs_error": round(float(difference.abs().max()), 8),
|
||||
"relative_l2": round(float(difference.norm() / upstream.norm()), 8),
|
||||
"cosine": round(
|
||||
float(torch.dot(upstream.double(), actual.double()) / (upstream.norm() * actual.norm())),
|
||||
8,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _load_vllm_payload(vllm_dir: Path, template: str) -> dict[str, object]:
|
||||
matches = []
|
||||
for path in vllm_dir.glob("breeze_*.pt"):
|
||||
payload = torch.load(path, weights_only=False)
|
||||
if payload.get("template") == template:
|
||||
matches.append(payload)
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"expected one vLLM dump for {template!r}, found {len(matches)}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _longest_exact_frame_prefix(upstream: torch.Tensor, actual: torch.Tensor) -> int:
|
||||
count = min(int(upstream.shape[0]), int(actual.shape[0]))
|
||||
equal = (upstream[:count] == actual[:count]).all(dim=-1)
|
||||
prefix = 0
|
||||
for is_equal in equal.tolist():
|
||||
if not is_equal:
|
||||
break
|
||||
prefix += 1
|
||||
return prefix
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--upstream-dir", type=Path, required=True)
|
||||
parser.add_argument("--vllm-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--plain-audio", type=Path)
|
||||
parser.add_argument("--instruction-audio", type=Path)
|
||||
parser.add_argument("--reference-audio", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
audio_paths = {
|
||||
"plain": args.plain_audio,
|
||||
"instruction": args.instruction_audio,
|
||||
"reference_edit": args.reference_audio,
|
||||
}
|
||||
report: dict[str, object] = {
|
||||
"upstream_dir": str(args.upstream_dir.resolve()),
|
||||
"vllm_dir": str(args.vllm_dir.resolve()),
|
||||
"cases": [],
|
||||
}
|
||||
for name, template in TEMPLATES.items():
|
||||
upstream_codes = torch.load(args.upstream_dir / name / "codes.pt", weights_only=False)
|
||||
upstream_prompt = torch.load(args.upstream_dir / name / "prompt_ids.pt", weights_only=False)
|
||||
actual_payload = _load_vllm_payload(args.vllm_dir, template)
|
||||
actual_codes = actual_payload["codes"]
|
||||
actual_prompt = actual_payload["prompt_ids"]
|
||||
common_frames = min(int(upstream_codes.shape[0]), int(actual_codes.shape[0]))
|
||||
code_equal = upstream_codes[:common_frames] == actual_codes[:common_frames]
|
||||
frame_equal = code_equal.all(dim=-1)
|
||||
prefill_metrics = None
|
||||
upstream_hidden_path = args.upstream_dir / name / "prefill_hidden.pt"
|
||||
if upstream_hidden_path.exists():
|
||||
prefill_files = [
|
||||
path
|
||||
for path in args.vllm_dir.glob("prefill_*.pt")
|
||||
if torch.load(path, weights_only=False).get("template") == template
|
||||
]
|
||||
if len(prefill_files) == 1:
|
||||
actual_hidden = torch.load(prefill_files[0], weights_only=False)["hidden"]
|
||||
prefill_metrics = _vector_metrics(
|
||||
torch.load(upstream_hidden_path, weights_only=False),
|
||||
actual_hidden,
|
||||
)
|
||||
entry: dict[str, object] = {
|
||||
"name": name,
|
||||
"template": template,
|
||||
"prompt": {
|
||||
"upstream_tokens": int(upstream_prompt.numel()),
|
||||
"vllm_tokens": int(actual_prompt.numel()),
|
||||
"exact": bool(torch.equal(upstream_prompt, actual_prompt)),
|
||||
},
|
||||
"frames": {
|
||||
"upstream": int(upstream_codes.shape[0]),
|
||||
"vllm": int(actual_codes.shape[0]),
|
||||
"codebooks": int(upstream_codes.shape[-1]),
|
||||
"exact_sequence": bool(torch.equal(upstream_codes, actual_codes)),
|
||||
"common_frames": common_frames,
|
||||
"exact_common_frames": int(frame_equal.sum()),
|
||||
"longest_exact_prefix": _longest_exact_frame_prefix(upstream_codes, actual_codes),
|
||||
"exact_codes": int(code_equal.sum()),
|
||||
"total_common_codes": int(code_equal.numel()),
|
||||
"code_accuracy": round(float(code_equal.float().mean()), 8),
|
||||
"codebook0_accuracy": round(
|
||||
float((upstream_codes[:common_frames, 0] == actual_codes[:common_frames, 0]).float().mean()),
|
||||
8,
|
||||
),
|
||||
"range_valid": bool(
|
||||
upstream_codes.min() >= 0
|
||||
and upstream_codes.max() < 2048
|
||||
and actual_codes.min() >= 0
|
||||
and actual_codes.max() < 2048
|
||||
),
|
||||
},
|
||||
"prefill_hidden": prefill_metrics,
|
||||
}
|
||||
if audio_paths[name] is not None and Path(audio_paths[name]).exists(): # type: ignore[arg-type]
|
||||
entry["audio"] = _audio_metrics(Path(audio_paths[name])) # type: ignore[arg-type]
|
||||
report["cases"].append(entry)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Generate deterministic Breeze-TTS-2 golden frames with the upstream runtime.
|
||||
|
||||
This tool is intentionally separate from vLLM-Omni: it imports the unmodified
|
||||
``breezeblue-ai/breeze-tts`` reference implementation, fixes greedy decoding,
|
||||
and writes both generated codec frames and decoded audio for offline parity
|
||||
checks. It is a diagnostic tool, not a serving path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
|
||||
def _plain_template(upstream_templates):
|
||||
"""The upstream CLI does not register tts_plain; reuse its segment builder."""
|
||||
return SimpleNamespace(
|
||||
name="tts_plain",
|
||||
required_fields=("text",),
|
||||
build_segments=upstream_templates._tts_plain_segments,
|
||||
build_negative_segments=None,
|
||||
build_dual_branches=None,
|
||||
)
|
||||
|
||||
|
||||
def _greedy(model) -> None:
|
||||
model.generation_config.do_sample = False
|
||||
model.generation_config.temperature = None
|
||||
model.generation_config.top_p = None
|
||||
model.generation_config.top_k = None
|
||||
depth = model.depth_decoder.generation_config
|
||||
depth.depth_decoder_do_sample = False
|
||||
depth.do_sample = False
|
||||
depth.temperature = None
|
||||
depth.top_p = None
|
||||
depth.top_k = None
|
||||
|
||||
|
||||
def _valid_frames(sequences: torch.Tensor, pad_token_id: int) -> torch.Tensor:
|
||||
frames = sequences[0].detach().cpu().to(torch.long)
|
||||
if frames.ndim == 1:
|
||||
frames = frames.unsqueeze(0)
|
||||
is_pad = (frames == pad_token_id).all(dim=-1)
|
||||
if bool(is_pad.any()):
|
||||
frames = frames[: int(torch.argmax(is_pad.int()).item())]
|
||||
return frames.contiguous()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--upstream", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--reference-audio", type=Path)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=32)
|
||||
args = parser.parse_args()
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
upstream_posix = str(args.upstream.resolve())
|
||||
if upstream_posix not in sys.path:
|
||||
sys.path.insert(0, upstream_posix)
|
||||
|
||||
from breeze_infer.runtime import load_runtime, resolve_device
|
||||
from breeze_infer.templates import get_template, prepare_inputs
|
||||
|
||||
device = resolve_device()
|
||||
# ``torch.manual_seed`` seeds CUDA RNGs as well, so no device-specific
|
||||
# seeding call is needed here.
|
||||
torch.manual_seed(0)
|
||||
tokenizer, model, audio_tokenizer = load_runtime(
|
||||
args.model,
|
||||
device=device,
|
||||
attn_implementation="eager",
|
||||
)
|
||||
# Do not enable Transformers 4.x's ``fix_mistral_regex`` compatibility
|
||||
# path: vLLM-Omni runs with Transformers 5.x, and direct tokenization checks
|
||||
# showed both runtimes produce the same token IDs without that patch.
|
||||
_greedy(model)
|
||||
model.eval()
|
||||
first_backbone_hidden: list[torch.Tensor] = []
|
||||
|
||||
def _capture_first_backbone_hidden(_module, _inputs, output):
|
||||
if first_backbone_hidden:
|
||||
return
|
||||
hidden = getattr(output, "last_hidden_state", None)
|
||||
if isinstance(hidden, torch.Tensor):
|
||||
first_backbone_hidden.append(hidden[0, -1].detach().cpu().to(torch.float32).contiguous())
|
||||
|
||||
model.backbone_model.register_forward_hook(_capture_first_backbone_hidden)
|
||||
|
||||
cases = [
|
||||
{
|
||||
"name": "plain",
|
||||
"template": _plain_template(sys.modules["breeze_infer.templates"]),
|
||||
"request": {
|
||||
"id": "golden-plain",
|
||||
"text": "Golden plain alignment check.",
|
||||
"speaker": "S0",
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "instruction",
|
||||
"template": get_template("tts_instruction"),
|
||||
"request": {
|
||||
"id": "golden-instruction",
|
||||
"text": "Golden instruction alignment check.",
|
||||
"instruction": "Speak calmly, warmly, and clearly.",
|
||||
"speaker": "S0",
|
||||
},
|
||||
},
|
||||
]
|
||||
if args.reference_audio is not None:
|
||||
cases.append(
|
||||
{
|
||||
"name": "reference_edit",
|
||||
"template": get_template("ref_edit_tata"),
|
||||
"request": {
|
||||
"id": "golden-reference-edit",
|
||||
"text": "Golden reference alignment check.",
|
||||
"instruction": "Speak calmly and clearly.",
|
||||
"ref_audio_path": str(args.reference_audio),
|
||||
"ref_text": "Hello from Breeze.",
|
||||
"speaker": "S0",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"runtime": "upstream breezeblue-ai/breeze-tts",
|
||||
"model": str(args.model.resolve()),
|
||||
"upstream": upstream_posix,
|
||||
"device": device,
|
||||
"seed": 0,
|
||||
"max_new_tokens": args.max_new_tokens,
|
||||
"decoding": "greedy",
|
||||
"cfg_scale": 1.0,
|
||||
"cases": [],
|
||||
}
|
||||
for case in cases:
|
||||
first_backbone_hidden.clear()
|
||||
inputs = prepare_inputs(
|
||||
tokenizer,
|
||||
audio_tokenizer,
|
||||
model,
|
||||
[case["request"]],
|
||||
case["template"],
|
||||
guidance_scale=1.0,
|
||||
guidance_scale_ref=None,
|
||||
guidance_scale_ins=None,
|
||||
)
|
||||
output = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=args.max_new_tokens,
|
||||
do_sample=False,
|
||||
return_dict_in_generate=True,
|
||||
output_audio=True,
|
||||
audio_tokenizer=audio_tokenizer,
|
||||
)
|
||||
frames = _valid_frames(output.sequences, model.config.codebook_pad_token_id)
|
||||
audio = output.audio[0].detach().cpu().to(torch.float32).reshape(-1)
|
||||
case_dir = args.output_dir / case["name"]
|
||||
case_dir.mkdir(parents=True, exist_ok=True)
|
||||
torch.save(frames, case_dir / "codes.pt")
|
||||
prompt_ids = inputs["input_ids"].detach().cpu().to(torch.long).reshape(-1)
|
||||
torch.save(prompt_ids, case_dir / "prompt_ids.pt")
|
||||
if first_backbone_hidden:
|
||||
torch.save(first_backbone_hidden[0], case_dir / "prefill_hidden.pt")
|
||||
sf.write(case_dir / "upstream.wav", audio.numpy(), 24_000, subtype="PCM_16")
|
||||
entry = {
|
||||
"name": case["name"],
|
||||
"request": case["request"],
|
||||
"prompt_tokens": int(prompt_ids.numel()),
|
||||
"frames": int(frames.shape[0]),
|
||||
"codebooks": int(frames.shape[-1]) if frames.ndim == 2 else 0,
|
||||
"min_code": int(frames.min().item()) if frames.numel() else None,
|
||||
"max_code": int(frames.max().item()) if frames.numel() else None,
|
||||
"audio_samples": int(audio.numel()),
|
||||
"audio_sha256": __import__("hashlib").sha256((case_dir / "upstream.wav").read_bytes()).hexdigest(),
|
||||
}
|
||||
manifest["cases"].append(entry)
|
||||
print(json.dumps(entry), flush=True)
|
||||
|
||||
(args.output_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user