[Feature] Support MOSS-TTS-Local-v1.5 Streaming (#4804)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
d8db832fc2
commit
a8b2bf11d2
15 files changed
+3222
-189
No files matched your search
@@ -36,7 +36,9 @@ Enable with ``SEED_TTS_WER_EVAL=1`` or ``--seed-tts-wer-eval``. Install optional
|
||||
pip install 'vllm-omni[dev]'
|
||||
|
||||
Env: ``SEED_TTS_EVAL_DEVICE`` (e.g. ``cuda:0``, ``cpu``); ``SEED_TTS_HF_WHISPER_MODEL``
|
||||
defaults to ``openai/whisper-large-v3`` (override for debugging only).
|
||||
defaults to ``openai/whisper-large-v3`` (override for debugging only). Set
|
||||
``SEED_TTS_WER_SAVE_AUDIO_DIR`` to save the captured 24 kHz mono WAV used by
|
||||
WER evaluation for each synthesized utterance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -50,6 +52,7 @@ import string
|
||||
import tempfile
|
||||
import threading
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -87,6 +90,30 @@ def pcm_s16le_mono_to_wav_bytes(pcm: bytes, *, sample_rate: int = 24000) -> byte
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _safe_filename_part(value: Any, *, default: str = "item", max_len: int = 96) -> str:
|
||||
text = str(value or "").strip() or default
|
||||
safe = "".join(ch if ch.isalnum() or ch in ("-", "_", ".") else "_" for ch in text)
|
||||
safe = safe.strip("._") or default
|
||||
return safe[:max_len]
|
||||
|
||||
|
||||
def _save_seed_tts_eval_audio(
|
||||
pcm: bytes,
|
||||
*,
|
||||
output_dir: Path | None,
|
||||
index: int,
|
||||
utterance_id: Any,
|
||||
locale: str,
|
||||
) -> str | None:
|
||||
if output_dir is None:
|
||||
return None
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = f"{index:05d}_{_safe_filename_part(utterance_id)}_{_safe_filename_part(locale)}"
|
||||
path = output_dir / f"{stem}.wav"
|
||||
path.write_bytes(pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000))
|
||||
return str(path)
|
||||
|
||||
|
||||
def _get_eval_device() -> str:
|
||||
explicit = os.environ.get("SEED_TTS_EVAL_DEVICE", "").strip()
|
||||
if explicit:
|
||||
@@ -512,13 +539,18 @@ def compute_seed_tts_wer_metrics(
|
||||
sim_skipped_no_ref = 0
|
||||
utmos_failed = 0
|
||||
utmos_on = _eval_submetric_enabled("SEED_TTS_UTMOS_EVAL", default=False)
|
||||
save_audio_raw = os.environ.get("SEED_TTS_WER_SAVE_AUDIO_DIR", "").strip()
|
||||
save_audio_dir = Path(save_audio_raw).expanduser() if save_audio_raw else None
|
||||
saved_audio = 0
|
||||
save_audio_failed = 0
|
||||
|
||||
for req, out in zip(input_requests, outputs, strict=True):
|
||||
for index, (req, out) in enumerate(zip(input_requests, outputs, strict=True)):
|
||||
assert isinstance(req, SeedTTSSampleRequest)
|
||||
ref = req.prompt
|
||||
locale = req.seed_tts_locale or "en"
|
||||
row_lang = "zh" if locale.lower().startswith("zh") else "en"
|
||||
utmos_v: float | None = None
|
||||
audio_path: str | None = None
|
||||
|
||||
if not out.success:
|
||||
request_failed += 1
|
||||
@@ -545,6 +577,23 @@ def compute_seed_tts_wer_metrics(
|
||||
}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
audio_path = _save_seed_tts_eval_audio(
|
||||
pcm,
|
||||
output_dir=save_audio_dir,
|
||||
index=index,
|
||||
utterance_id=req.seed_tts_utterance_id,
|
||||
locale=locale,
|
||||
)
|
||||
if audio_path:
|
||||
saved_audio += 1
|
||||
except OSError as e:
|
||||
save_audio_failed += 1
|
||||
logger.warning(
|
||||
"Seed-TTS WER audio save failed for utterance=%s: %s",
|
||||
req.seed_tts_utterance_id,
|
||||
e,
|
||||
)
|
||||
|
||||
wav_16k = _pcm_s16le_to_f32_16k(pcm)
|
||||
if len(wav_16k) == 0:
|
||||
@@ -555,6 +604,7 @@ def compute_seed_tts_wer_metrics(
|
||||
"utterance_id": req.seed_tts_utterance_id,
|
||||
"locale": locale,
|
||||
"error": "empty_audio",
|
||||
"audio_path": audio_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -609,6 +659,7 @@ def compute_seed_tts_wer_metrics(
|
||||
"locale": locale,
|
||||
"error": "asr_exception",
|
||||
"detail": str(e)[:500],
|
||||
"audio_path": audio_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -621,6 +672,7 @@ def compute_seed_tts_wer_metrics(
|
||||
"utterance_id": req.seed_tts_utterance_id,
|
||||
"locale": locale,
|
||||
"error": "empty_asr",
|
||||
"audio_path": audio_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -637,6 +689,7 @@ def compute_seed_tts_wer_metrics(
|
||||
"locale": locale,
|
||||
"error": "wer_compute_failed",
|
||||
"detail": str(e)[:500],
|
||||
"audio_path": audio_path,
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -673,6 +726,8 @@ def compute_seed_tts_wer_metrics(
|
||||
"reference_raw": raw_truth,
|
||||
"asr_raw": raw_hypo,
|
||||
}
|
||||
if audio_path:
|
||||
row["audio_path"] = audio_path
|
||||
if sim_v is not None:
|
||||
row["sim"] = sim_v
|
||||
if utmos_v is not None:
|
||||
@@ -697,7 +752,11 @@ def compute_seed_tts_wer_metrics(
|
||||
"seed_tts_utmos_mean": statistics.fmean(utmos_values) if utmos_values else None,
|
||||
"seed_tts_utmos_median": statistics.median(utmos_values) if utmos_values else None,
|
||||
"seed_tts_utmos_failed": utmos_failed,
|
||||
"seed_tts_saved_audio": saved_audio,
|
||||
"seed_tts_save_audio_failed": save_audio_failed,
|
||||
}
|
||||
if save_audio_dir is not None:
|
||||
result["seed_tts_save_audio_dir"] = str(save_audio_dir)
|
||||
if include_per_item:
|
||||
result["seed_tts_wer_eval_items"] = items
|
||||
return result
|
||||
@@ -729,6 +788,13 @@ def print_seed_tts_wer_summary(metrics: dict[str, Any]) -> None:
|
||||
print("{:<40} {:<10}".format("Request failed:", metrics.get("seed_tts_request_failed", 0)))
|
||||
print("{:<40} {:<10}".format("No PCM captured:", metrics.get("seed_tts_no_pcm", 0)))
|
||||
print("{:<40} {:<10}".format("ASR / WER failed:", metrics.get("seed_tts_asr_failed", 0)))
|
||||
save_dir = metrics.get("seed_tts_save_audio_dir")
|
||||
if save_dir:
|
||||
print("{:<40} {:<10}".format("Saved eval WAVs:", metrics.get("seed_tts_saved_audio", 0)))
|
||||
print("{:<40} {}".format("Saved eval WAV dir:", save_dir))
|
||||
failed = int(metrics.get("seed_tts_save_audio_failed", 0) or 0)
|
||||
if failed:
|
||||
print("{:<40} {:<10}".format("Save eval WAV errors:", failed))
|
||||
if sim_ev or metrics.get("seed_tts_sim_skipped_no_ref") or metrics.get("seed_tts_sim_failed"):
|
||||
print("{:<40} {:<10}".format("SIM evaluated (higher ~ closer):", sim_ev))
|
||||
sm = metrics.get("seed_tts_sim_mean")
|
||||
|
||||
Reference in new issue
Block a user