diff --git a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py index efa312dec..70ed7cfef 100644 --- a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py +++ b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py @@ -39,6 +39,8 @@ Env: ``SEED_TTS_EVAL_DEVICE`` (e.g. ``cuda:0``, ``cpu``); ``SEED_TTS_HF_WHISPER_ defaults to ``openai/whisper-large-v3`` (override for debugging only). Set ``SEED_TTS_WER_SAVE_AUDIO_DIR`` to save the captured 24 kHz mono WAV used by WER evaluation for each synthesized utterance. +Streaming PCM is decoded using ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` / +``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` (default: 24 kHz mono). """ from __future__ import annotations @@ -59,6 +61,7 @@ import numpy as np from vllm.benchmarks.datasets import SampleRequest from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSSampleRequest +from vllm_omni.metrics.definitions import stream_pcm_format_from_env logger = logging.getLogger(__name__) @@ -173,12 +176,27 @@ def process_one_official(hypo: str, truth: str, lang: str) -> tuple[float, str, return wer, raw_truth, raw_hypo -def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000) -> np.ndarray: +def _pcm_s16le_to_f32_16k( + pcm: bytes, + pcm_sample_rate: int | None = None, + channels: int | None = None, +) -> np.ndarray: import scipy.signal if not pcm: return np.zeros(0, dtype=np.float32) + if pcm_sample_rate is None or channels is None: + env_sample_rate, env_channels = stream_pcm_format_from_env() + if pcm_sample_rate is None: + pcm_sample_rate = env_sample_rate + if channels is None: + channels = env_channels raw = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + if channels > 1: + usable = (len(raw) // channels) * channels + if usable <= 0: + return np.zeros(0, dtype=np.float32) + raw = raw[:usable].reshape(-1, channels).mean(axis=1) target_len = int(len(raw) * 16000 / pcm_sample_rate) if target_len <= 0: return np.zeros(0, dtype=np.float32) diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index d6e28319c..a4eb683e6 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -35,8 +35,6 @@ from vllm.benchmarks.lib.endpoint_request_func import ( from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike -logger = init_logger(__name__) - from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset @@ -51,9 +49,9 @@ from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDa from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset from vllm_omni.metrics import definitions as defs +logger = init_logger(__name__) + _AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S" -_AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE" -_AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS" RETURN_STAGE_METRICS_FIELD = "return_stage_metrics" _IMAGE_STAGE_METRICS_BACKENDS = frozenset({"openai-image-edits-omni"}) _PRINT_STAGE = False @@ -111,32 +109,6 @@ def _audio_continuity_threshold_s() -> float: return max(value, 0.0) -def _audio_pcm_format() -> tuple[int, int]: - """Return ``(sample_rate, channels)`` of the streamed PCM response. - - Defaults to 24 kHz mono (Qwen3-TTS / VoxCPM2 / most MOSS-TTS variants). - Override for models with a different codec output format (e.g. - MOSS-TTS-Local-Transformer-v1.5, which streams 48 kHz stereo PCM) via - ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` / ``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` - - getting this wrong silently skews audio_duration/audio_rtf/underrun stats. - """ - sample_rate = 24000 - channels = 1 - raw_sr = os.environ.get(_AUDIO_SAMPLE_RATE_ENV) - if raw_sr: - try: - sample_rate = int(raw_sr) - except ValueError: - logger.warning("Invalid %s=%r; using default %d", _AUDIO_SAMPLE_RATE_ENV, raw_sr, sample_rate) - raw_ch = os.environ.get(_AUDIO_CHANNELS_ENV) - if raw_ch: - try: - channels = int(raw_ch) - except ValueError: - logger.warning("Invalid %s=%r; using default %d", _AUDIO_CHANNELS_ENV, raw_ch, channels) - return sample_rate, channels - - def _pcm_s16le_to_seed_tts_wer_bytes( pcm_bytes: bytes, *, @@ -1187,9 +1159,8 @@ async def async_request_openai_audio_speech( output = MixRequestFuncOutput() output.prompt_len = request_func_input.prompt_len - # PCM format: 16-bit signed; sample_rate/channels are model-dependent - # (see _audio_pcm_format docstring). - sample_rate, channels = _audio_pcm_format() + # PCM format: 16-bit signed; sample_rate/channels are model-dependent. + sample_rate, channels = defs.stream_pcm_format_from_env() sample_width = 2 # 16-bit = 2 bytes st = time.perf_counter() diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 78bb0ab2c..72654738f 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -2620,16 +2620,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self, request, *, - include_language=False, include_voice=False, plain_text_passthrough=False, ): instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None instruction_dict: dict[str, Any] = {} - if include_language and request.language not in (None, "", "Auto"): - instruction_dict["方言"] = request.language - voice_lower = request.voice.lower() if isinstance(request.voice, str) else None if include_voice and request.voice and not (voice_lower and voice_lower in self.uploaded_speakers): instruction_dict["IP"] = request.voice @@ -2652,7 +2648,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """Build a Ming instruction payload from OpenAI speech fields.""" return self._parse_ming_instruction_fields( request, - include_language=True, include_voice=True, plain_text_passthrough=True, ) diff --git a/vllm_omni/metrics/definitions.py b/vllm_omni/metrics/definitions.py index 3b7e6e8d8..9660e4f34 100644 --- a/vllm_omni/metrics/definitions.py +++ b/vllm_omni/metrics/definitions.py @@ -11,11 +11,14 @@ time-bearing metrics use the ``_s`` suffix (values in seconds), counters use ``_total`` (auto-suffixed by the prometheus client), sizes use ``_bytes``. """ +import logging +import os from typing import Any # vllm_omni: namespace for omni-specific Prometheus families, distinct from # the upstream vllm:* families. METRIC_PREFIX = "vllm_omni:" +logger = logging.getLogger(__name__) # ============================================================================ @@ -280,6 +283,9 @@ def compute_denoise_step_latency(stage_gen_time: float, num_inference_steps: int # ming_flash 16000 — these models populate multimodal_output["audio_sample_rate"] # at runtime so this default only kicks in when the field is missing. DEFAULT_AUDIO_SAMPLE_RATE = 24000 +DEFAULT_AUDIO_CHANNELS = 1 +AUDIO_SAMPLE_RATE_ENV = "VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE" +AUDIO_CHANNELS_ENV = "VLLM_OMNI_BENCH_AUDIO_CHANNELS" _SAMPLE_RATE_KEYS = ("output_sample_rate", "audio_sample_rate", "sample_rate", "sampling_rate", "sr") @@ -306,3 +312,26 @@ def resolve_audio_sample_rate(source: dict[str, Any] | Any | None) -> int: if value > 0: return value return DEFAULT_AUDIO_SAMPLE_RATE + + +def stream_pcm_format_from_env( + *, + default_sample_rate: int = DEFAULT_AUDIO_SAMPLE_RATE, + default_channels: int = DEFAULT_AUDIO_CHANNELS, +) -> tuple[int, int]: + """Return the sample rate and channel count for streamed raw PCM.""" + sample_rate = default_sample_rate + channels = default_channels + raw_sr = os.environ.get(AUDIO_SAMPLE_RATE_ENV) + if raw_sr: + try: + sample_rate = int(raw_sr) + except ValueError: + logger.warning("Invalid %s=%r; using default %d", AUDIO_SAMPLE_RATE_ENV, raw_sr, sample_rate) + raw_ch = os.environ.get(AUDIO_CHANNELS_ENV) + if raw_ch: + try: + channels = int(raw_ch) + except ValueError: + logger.warning("Invalid %s=%r; using default %d", AUDIO_CHANNELS_ENV, raw_ch, channels) + return max(sample_rate, 1), max(channels, 1) diff --git a/vllm_omni/model_executor/models/common/ming/audio_vae.py b/vllm_omni/model_executor/models/common/ming/audio_vae.py index 89520f515..aecc35da9 100644 --- a/vllm_omni/model_executor/models/common/ming/audio_vae.py +++ b/vllm_omni/model_executor/models/common/ming/audio_vae.py @@ -138,8 +138,8 @@ class Encoder(nn.Module): self.norm = nn.LayerNorm(config.hidden_size) self.patch_size = patch_size if patch_size != -1: - aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4}) - self.aggregator = Qwen2Model(aggregator_config) + config.num_hidden_layers = 4 + self.aggregator = Qwen2Model(config) self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size)) self.cls_embed.data.normal_(0, 0.02) @@ -346,8 +346,9 @@ class AudioVAE(PreTrainedModel): h, _ = self.encoder(waveform) h = h.transpose(1, 2) - mean = torch.chunk(h, 2, dim=1)[0] - latent = mean.transpose(1, 2) + mean, scale = torch.chunk(h, 2, dim=1) + std = F.softplus(scale) + 1e-4 + latent = (mean + std * torch.randn_like(mean)).transpose(1, 2) return latent, frame_num @torch.inference_mode() diff --git a/vllm_omni/model_executor/models/ming_tts/patch_emission.py b/vllm_omni/model_executor/models/ming_tts/patch_emission.py index c0a25a1c1..89acdf231 100644 --- a/vllm_omni/model_executor/models/ming_tts/patch_emission.py +++ b/vllm_omni/model_executor/models/ming_tts/patch_emission.py @@ -127,7 +127,7 @@ def _validate_ming_decode_window( continue max_steps = _resolve_runtime_int(info, KEY_MAX_DECODE_STEPS, default_max_decode_steps) min_steps = _resolve_optional_runtime_int(info, KEY_MIN_DECODE_STEPS, 0) - min_required = max(min_stop_step + 1, min_steps) + min_required = max(min_stop_step + 2, min_steps) if max_steps < min_required: req_id = info.get(KEY_REQUEST_ID, f"idx={i}") raise ValueError( @@ -148,7 +148,7 @@ def _resolve_ming_stop_decision( audio_dummy_token_id: int, text_eos_token_id: int, ) -> tuple[str, bool, bool, int, int]: - min_required_decode_steps = max(min_stop_step + 1, min_decode_steps) + min_required_decode_steps = max(min_stop_step + 2, min_decode_steps) if max_decode_steps < min_required_decode_steps: raise RuntimeError( "Invalid Ming decode window: "