[BugFix][Ming-TTS] Improve Seed-TTS WER accuracy (#4859)
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com>
This commit is contained in:
1 parent
a8b2bf11d2
commit
991fb7408b
6 files changed
+59
-45
No files matched your search
@@ -39,6 +39,8 @@ Env: ``SEED_TTS_EVAL_DEVICE`` (e.g. ``cuda:0``, ``cpu``); ``SEED_TTS_HF_WHISPER_
|
||||
defaults to ``openai/whisper-large-v3`` (override for debugging only). Set
|
||||
``SEED_TTS_WER_SAVE_AUDIO_DIR`` to save the captured 24 kHz mono WAV used by
|
||||
WER evaluation for each synthesized utterance.
|
||||
Streaming PCM is decoded using ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` /
|
||||
``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` (default: 24 kHz mono).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -59,6 +61,7 @@ import numpy as np
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSSampleRequest
|
||||
from vllm_omni.metrics.definitions import stream_pcm_format_from_env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -173,12 +176,27 @@ def process_one_official(hypo: str, truth: str, lang: str) -> tuple[float, str,
|
||||
return wer, raw_truth, raw_hypo
|
||||
|
||||
|
||||
def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000) -> np.ndarray:
|
||||
def _pcm_s16le_to_f32_16k(
|
||||
pcm: bytes,
|
||||
pcm_sample_rate: int | None = None,
|
||||
channels: int | None = None,
|
||||
) -> np.ndarray:
|
||||
import scipy.signal
|
||||
|
||||
if not pcm:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
if pcm_sample_rate is None or channels is None:
|
||||
env_sample_rate, env_channels = stream_pcm_format_from_env()
|
||||
if pcm_sample_rate is None:
|
||||
pcm_sample_rate = env_sample_rate
|
||||
if channels is None:
|
||||
channels = env_channels
|
||||
raw = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
if channels > 1:
|
||||
usable = (len(raw) // channels) * channels
|
||||
if usable <= 0:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
raw = raw[:usable].reshape(-1, channels).mean(axis=1)
|
||||
target_len = int(len(raw) * 16000 / pcm_sample_rate)
|
||||
if target_len <= 0:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
|
||||
@@ -35,8 +35,6 @@ from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
|
||||
from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
|
||||
@@ -51,9 +49,9 @@ from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDa
|
||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||
from vllm_omni.metrics import definitions as defs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
|
||||
_AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE"
|
||||
_AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS"
|
||||
RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
|
||||
_IMAGE_STAGE_METRICS_BACKENDS = frozenset({"openai-image-edits-omni"})
|
||||
_PRINT_STAGE = False
|
||||
@@ -111,32 +109,6 @@ def _audio_continuity_threshold_s() -> float:
|
||||
return max(value, 0.0)
|
||||
|
||||
|
||||
def _audio_pcm_format() -> tuple[int, int]:
|
||||
"""Return ``(sample_rate, channels)`` of the streamed PCM response.
|
||||
|
||||
Defaults to 24 kHz mono (Qwen3-TTS / VoxCPM2 / most MOSS-TTS variants).
|
||||
Override for models with a different codec output format (e.g.
|
||||
MOSS-TTS-Local-Transformer-v1.5, which streams 48 kHz stereo PCM) via
|
||||
``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` / ``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` -
|
||||
getting this wrong silently skews audio_duration/audio_rtf/underrun stats.
|
||||
"""
|
||||
sample_rate = 24000
|
||||
channels = 1
|
||||
raw_sr = os.environ.get(_AUDIO_SAMPLE_RATE_ENV)
|
||||
if raw_sr:
|
||||
try:
|
||||
sample_rate = int(raw_sr)
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", _AUDIO_SAMPLE_RATE_ENV, raw_sr, sample_rate)
|
||||
raw_ch = os.environ.get(_AUDIO_CHANNELS_ENV)
|
||||
if raw_ch:
|
||||
try:
|
||||
channels = int(raw_ch)
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", _AUDIO_CHANNELS_ENV, raw_ch, channels)
|
||||
return sample_rate, channels
|
||||
|
||||
|
||||
def _pcm_s16le_to_seed_tts_wer_bytes(
|
||||
pcm_bytes: bytes,
|
||||
*,
|
||||
@@ -1187,9 +1159,8 @@ async def async_request_openai_audio_speech(
|
||||
output = MixRequestFuncOutput()
|
||||
output.prompt_len = request_func_input.prompt_len
|
||||
|
||||
# PCM format: 16-bit signed; sample_rate/channels are model-dependent
|
||||
# (see _audio_pcm_format docstring).
|
||||
sample_rate, channels = _audio_pcm_format()
|
||||
# PCM format: 16-bit signed; sample_rate/channels are model-dependent.
|
||||
sample_rate, channels = defs.stream_pcm_format_from_env()
|
||||
sample_width = 2 # 16-bit = 2 bytes
|
||||
|
||||
st = time.perf_counter()
|
||||
|
||||
@@ -2620,16 +2620,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self,
|
||||
request,
|
||||
*,
|
||||
include_language=False,
|
||||
include_voice=False,
|
||||
plain_text_passthrough=False,
|
||||
):
|
||||
instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None
|
||||
instruction_dict: dict[str, Any] = {}
|
||||
|
||||
if include_language and request.language not in (None, "", "Auto"):
|
||||
instruction_dict["方言"] = request.language
|
||||
|
||||
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
|
||||
if include_voice and request.voice and not (voice_lower and voice_lower in self.uploaded_speakers):
|
||||
instruction_dict["IP"] = request.voice
|
||||
@@ -2652,7 +2648,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"""Build a Ming instruction payload from OpenAI speech fields."""
|
||||
return self._parse_ming_instruction_fields(
|
||||
request,
|
||||
include_language=True,
|
||||
include_voice=True,
|
||||
plain_text_passthrough=True,
|
||||
)
|
||||
|
||||
@@ -11,11 +11,14 @@ time-bearing metrics use the ``_s`` suffix (values in seconds), counters use
|
||||
``_total`` (auto-suffixed by the prometheus client), sizes use ``_bytes``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
# vllm_omni: namespace for omni-specific Prometheus families, distinct from
|
||||
# the upstream vllm:* families.
|
||||
METRIC_PREFIX = "vllm_omni:"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -280,6 +283,9 @@ def compute_denoise_step_latency(stage_gen_time: float, num_inference_steps: int
|
||||
# ming_flash 16000 — these models populate multimodal_output["audio_sample_rate"]
|
||||
# at runtime so this default only kicks in when the field is missing.
|
||||
DEFAULT_AUDIO_SAMPLE_RATE = 24000
|
||||
DEFAULT_AUDIO_CHANNELS = 1
|
||||
AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE"
|
||||
AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS"
|
||||
|
||||
_SAMPLE_RATE_KEYS = ("output_sample_rate", "audio_sample_rate", "sample_rate", "sampling_rate", "sr")
|
||||
|
||||
@@ -306,3 +312,26 @@ def resolve_audio_sample_rate(source: dict[str, Any] | Any | None) -> int:
|
||||
if value > 0:
|
||||
return value
|
||||
return DEFAULT_AUDIO_SAMPLE_RATE
|
||||
|
||||
|
||||
def stream_pcm_format_from_env(
|
||||
*,
|
||||
default_sample_rate: int = DEFAULT_AUDIO_SAMPLE_RATE,
|
||||
default_channels: int = DEFAULT_AUDIO_CHANNELS,
|
||||
) -> tuple[int, int]:
|
||||
"""Return the sample rate and channel count for streamed raw PCM."""
|
||||
sample_rate = default_sample_rate
|
||||
channels = default_channels
|
||||
raw_sr = os.environ.get(AUDIO_SAMPLE_RATE_ENV)
|
||||
if raw_sr:
|
||||
try:
|
||||
sample_rate = int(raw_sr)
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", AUDIO_SAMPLE_RATE_ENV, raw_sr, sample_rate)
|
||||
raw_ch = os.environ.get(AUDIO_CHANNELS_ENV)
|
||||
if raw_ch:
|
||||
try:
|
||||
channels = int(raw_ch)
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", AUDIO_CHANNELS_ENV, raw_ch, channels)
|
||||
return max(sample_rate, 1), max(channels, 1)
|
||||
@@ -138,8 +138,8 @@ class Encoder(nn.Module):
|
||||
self.norm = nn.LayerNorm(config.hidden_size)
|
||||
self.patch_size = patch_size
|
||||
if patch_size != -1:
|
||||
aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4})
|
||||
self.aggregator = Qwen2Model(aggregator_config)
|
||||
config.num_hidden_layers = 4
|
||||
self.aggregator = Qwen2Model(config)
|
||||
self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
|
||||
self.cls_embed.data.normal_(0, 0.02)
|
||||
|
||||
@@ -346,8 +346,9 @@ class AudioVAE(PreTrainedModel):
|
||||
h, _ = self.encoder(waveform)
|
||||
h = h.transpose(1, 2)
|
||||
|
||||
mean = torch.chunk(h, 2, dim=1)[0]
|
||||
latent = mean.transpose(1, 2)
|
||||
mean, scale = torch.chunk(h, 2, dim=1)
|
||||
std = F.softplus(scale) + 1e-4
|
||||
latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)
|
||||
return latent, frame_num
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -127,7 +127,7 @@ def _validate_ming_decode_window(
|
||||
continue
|
||||
max_steps = _resolve_runtime_int(info, KEY_MAX_DECODE_STEPS, default_max_decode_steps)
|
||||
min_steps = _resolve_optional_runtime_int(info, KEY_MIN_DECODE_STEPS, 0)
|
||||
min_required = max(min_stop_step + 1, min_steps)
|
||||
min_required = max(min_stop_step + 2, min_steps)
|
||||
if max_steps < min_required:
|
||||
req_id = info.get(KEY_REQUEST_ID, f"idx={i}")
|
||||
raise ValueError(
|
||||
@@ -148,7 +148,7 @@ def _resolve_ming_stop_decision(
|
||||
audio_dummy_token_id: int,
|
||||
text_eos_token_id: int,
|
||||
) -> tuple[str, bool, bool, int, int]:
|
||||
min_required_decode_steps = max(min_stop_step + 1, min_decode_steps)
|
||||
min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)
|
||||
if max_decode_steps < min_required_decode_steps:
|
||||
raise RuntimeError(
|
||||
"Invalid Ming decode window: "
|
||||
|
||||
Reference in new issue
Block a user