[Feature] Support MOSS-TTS-Local-v1.5 Streaming (#4804)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
Canlin Guo authored and GitHub committed 2026-07-03 16:00:30 +08:00
1 parent d8db832fc2
commit a8b2bf11d2
15 files changed
+3222 -189

No files matched your search

@@ -36,7 +36,9 @@ Enable with ``SEED_TTS_WER_EVAL=1`` or ``--seed-tts-wer-eval``. Install optional
pip install 'vllm-omni[dev]'
Env: ``SEED_TTS_EVAL_DEVICE`` (e.g. ``cuda:0``, ``cpu``); ``SEED_TTS_HF_WHISPER_MODEL``
defaults to ``openai/whisper-large-v3`` (override for debugging only).
defaults to ``openai/whisper-large-v3`` (override for debugging only). Set
``SEED_TTS_WER_SAVE_AUDIO_DIR`` to save the captured 24 kHz mono WAV used by
WER evaluation for each synthesized utterance.
"""
from __future__ import annotations
@@ -50,6 +52,7 @@ import string
import tempfile
import threading
import wave
from pathlib import Path
from typing import Any
import numpy as np
@@ -87,6 +90,30 @@ def pcm_s16le_mono_to_wav_bytes(pcm: bytes, *, sample_rate: int = 24000) -> byte
return buf.getvalue()
def _safe_filename_part(value: Any, *, default: str = "item", max_len: int = 96) -> str:
text = str(value or "").strip() or default
safe = "".join(ch if ch.isalnum() or ch in ("-", "_", ".") else "_" for ch in text)
safe = safe.strip("._") or default
return safe[:max_len]
def _save_seed_tts_eval_audio(
pcm: bytes,
*,
output_dir: Path | None,
index: int,
utterance_id: Any,
locale: str,
) -> str | None:
if output_dir is None:
return None
output_dir.mkdir(parents=True, exist_ok=True)
stem = f"{index:05d}_{_safe_filename_part(utterance_id)}_{_safe_filename_part(locale)}"
path = output_dir / f"{stem}.wav"
path.write_bytes(pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000))
return str(path)
def _get_eval_device() -> str:
explicit = os.environ.get("SEED_TTS_EVAL_DEVICE", "").strip()
if explicit:
@@ -512,13 +539,18 @@ def compute_seed_tts_wer_metrics(
sim_skipped_no_ref = 0
utmos_failed = 0
utmos_on = _eval_submetric_enabled("SEED_TTS_UTMOS_EVAL", default=False)
save_audio_raw = os.environ.get("SEED_TTS_WER_SAVE_AUDIO_DIR", "").strip()
save_audio_dir = Path(save_audio_raw).expanduser() if save_audio_raw else None
saved_audio = 0
save_audio_failed = 0
for req, out in zip(input_requests, outputs, strict=True):
for index, (req, out) in enumerate(zip(input_requests, outputs, strict=True)):
assert isinstance(req, SeedTTSSampleRequest)
ref = req.prompt
locale = req.seed_tts_locale or "en"
row_lang = "zh" if locale.lower().startswith("zh") else "en"
utmos_v: float | None = None
audio_path: str | None = None
if not out.success:
request_failed += 1
@@ -545,6 +577,23 @@ def compute_seed_tts_wer_metrics(
}
)
continue
try:
audio_path = _save_seed_tts_eval_audio(
pcm,
output_dir=save_audio_dir,
index=index,
utterance_id=req.seed_tts_utterance_id,
locale=locale,
)
if audio_path:
saved_audio += 1
except OSError as e:
save_audio_failed += 1
logger.warning(
"Seed-TTS WER audio save failed for utterance=%s: %s",
req.seed_tts_utterance_id,
e,
)
wav_16k = _pcm_s16le_to_f32_16k(pcm)
if len(wav_16k) == 0:
@@ -555,6 +604,7 @@ def compute_seed_tts_wer_metrics(
"utterance_id": req.seed_tts_utterance_id,
"locale": locale,
"error": "empty_audio",
"audio_path": audio_path,
}
)
continue
@@ -609,6 +659,7 @@ def compute_seed_tts_wer_metrics(
"locale": locale,
"error": "asr_exception",
"detail": str(e)[:500],
"audio_path": audio_path,
}
)
continue
@@ -621,6 +672,7 @@ def compute_seed_tts_wer_metrics(
"utterance_id": req.seed_tts_utterance_id,
"locale": locale,
"error": "empty_asr",
"audio_path": audio_path,
}
)
continue
@@ -637,6 +689,7 @@ def compute_seed_tts_wer_metrics(
"locale": locale,
"error": "wer_compute_failed",
"detail": str(e)[:500],
"audio_path": audio_path,
}
)
continue
@@ -673,6 +726,8 @@ def compute_seed_tts_wer_metrics(
"reference_raw": raw_truth,
"asr_raw": raw_hypo,
}
if audio_path:
row["audio_path"] = audio_path
if sim_v is not None:
row["sim"] = sim_v
if utmos_v is not None:
@@ -697,7 +752,11 @@ def compute_seed_tts_wer_metrics(
"seed_tts_utmos_mean": statistics.fmean(utmos_values) if utmos_values else None,
"seed_tts_utmos_median": statistics.median(utmos_values) if utmos_values else None,
"seed_tts_utmos_failed": utmos_failed,
"seed_tts_saved_audio": saved_audio,
"seed_tts_save_audio_failed": save_audio_failed,
}
if save_audio_dir is not None:
result["seed_tts_save_audio_dir"] = str(save_audio_dir)
if include_per_item:
result["seed_tts_wer_eval_items"] = items
return result
@@ -729,6 +788,13 @@ def print_seed_tts_wer_summary(metrics: dict[str, Any]) -> None:
print("{:<40} {:<10}".format("Request failed:", metrics.get("seed_tts_request_failed", 0)))
print("{:<40} {:<10}".format("No PCM captured:", metrics.get("seed_tts_no_pcm", 0)))
print("{:<40} {:<10}".format("ASR / WER failed:", metrics.get("seed_tts_asr_failed", 0)))
save_dir = metrics.get("seed_tts_save_audio_dir")
if save_dir:
print("{:<40} {:<10}".format("Saved eval WAVs:", metrics.get("seed_tts_saved_audio", 0)))
print("{:<40} {}".format("Saved eval WAV dir:", save_dir))
failed = int(metrics.get("seed_tts_save_audio_failed", 0) or 0)
if failed:
print("{:<40} {:<10}".format("Save eval WAV errors:", failed))
if sim_ev or metrics.get("seed_tts_sim_skipped_no_ref") or metrics.get("seed_tts_sim_failed"):
print("{:<40} {:<10}".format("SIM evaluated (higher ~ closer):", sim_ev))
sm = metrics.get("seed_tts_sim_mean")