From 1d77e96d4de4bfd701bdb7b88113548ac0593530 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Thu, 24 Sep 2026 00:29:22 -0700 Subject: [PATCH] [Bugfix] Validate Breeze PCM speech content and calibrate HNR guard (#8044) Signed-off-by: Yueqian Lin --- tests/e2e/online_serving/test_breeze_tts_2.py | 8 +- tests/helpers/assertions.py | 48 ++++- tests/helpers/client.py | 2 + tests/helpers/tests/test_assertions.py | 170 ++++++++++++++++++ 4 files changed, 216 insertions(+), 12 deletions(-) diff --git a/tests/e2e/online_serving/test_breeze_tts_2.py b/tests/e2e/online_serving/test_breeze_tts_2.py index 6bb5b86dc..3d104f464 100644 --- a/tests/e2e/online_serving/test_breeze_tts_2.py +++ b/tests/e2e/online_serving/test_breeze_tts_2.py @@ -52,14 +52,16 @@ def test_english_streaming(omni_server, online_client, run_level) -> None: "input": "Hello, this is a demonstration of natural speech synthesis.", "instructions": "A warm, clear female voice speaking English at a moderate pace.", "transcript_language": "en", + "transcript_pcm_sample_rate": 24000, + "transcript_escalation_model": "large-v3", "response_format": "pcm", "sample_rate": 24000, "min_audio_bytes": 24000, "stream": True, "stream_format": "audio", - # The reference eager runtime scores 0.56 dB on this same prompt - # (seed 42, CFG=1); the shared 1 dB floor rejects it as well. - "min_hnr_db": 0.0, + # This fixed prompt scores -0.11 dB on L4 and 0.56 dB on H100. + # Keep a small HNR margin and also verify the spoken words above. + "min_hnr_db": -1.0, "seed": 42, "max_new_tokens": 8 if run_level == "core_model" else 250, } diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index ba1053928..d877c9068 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -688,7 +688,9 @@ def _compute_pcm_hnr_db(pcm_samples: np.ndarray, sr: int = _PCM_SPEECH_SAMPLE_RA peak = float(np.max(ac[min_lag:max_lag])) if 0 < peak < 1: hnr_values.append(10 * np.log10(peak / (1 - peak + 1e-10))) - return float(np.mean(hnr_values)) if hnr_values else 0.0 + # Silence and constant signals provide no usable harmonic estimate. They + # must not pass a model-specific floor of zero or below. + return float(np.mean(hnr_values)) if hnr_values else float("-inf") def _assert_pcm_int16_speech_hnr( @@ -714,10 +716,7 @@ def _assert_pcm_int16_speech_hnr( pcm_samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0 hnr = _compute_pcm_hnr_db(pcm_samples, sr=sr) print(f"PCM speech HNR: {hnr:.2f} dB (threshold: {min_hnr_db} dB, sr={sr})") - assert hnr >= min_hnr_db, ( - f"Audio distortion detected: HNR={hnr:.2f} dB < {min_hnr_db} dB. " - "Voice clone decoder may be losing ref_code speaker context on later chunks." - ) + assert hnr >= min_hnr_db, f"PCM speech HNR={hnr:.2f} dB is below the configured floor of {min_hnr_db} dB." def _response_has_audio_output(response: Any) -> bool: @@ -754,11 +753,38 @@ def _omni_assertion_needs_audio_transcript(request_config: dict[str, Any], run_l def _speech_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str | None) -> bool: if run_level not in {"advanced_model", "full_model"}: return False - if request_config.get("response_format") == "pcm": + if request_config.get("response_format") == "pcm" and "transcript_pcm_sample_rate" not in request_config: return False return bool(request_config.get("transcript_expected_text", request_config.get("input"))) +def _speech_pcm_sample_rate(request_config: dict[str, Any]) -> int: + expected = request_config.get("expected_sample_rate") + if "transcript_pcm_sample_rate" in request_config: + sample_rate = request_config["transcript_pcm_sample_rate"] + assert type(sample_rate) is int and sample_rate > 0, "transcript_pcm_sample_rate must be a positive integer" + assert expected is None or int(expected) == sample_rate, ( + "transcript_pcm_sample_rate must match expected_sample_rate" + ) + return sample_rate + return int(expected or _PCM_SPEECH_SAMPLE_RATE_HZ) + + +def _speech_audio_for_transcription(audio_bytes: bytes | None, request_config: dict[str, Any]) -> bytes | None: + """Give ASR a WAV container when a speech test opts into raw PCM transcription.""" + if not audio_bytes or request_config.get("response_format") != "pcm": + return audio_bytes + sample_rate = _speech_pcm_sample_rate(request_config) + assert len(audio_bytes) % 2 == 0, "PCM byte length must be aligned to int16" + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate) + wav_file.writeframes(audio_bytes) + return buffer.getvalue() + + def _resolve_audio_transcript( response: Any, request_config: dict[str, Any], @@ -777,7 +803,11 @@ def _resolve_audio_transcript( existing = getattr(response, "audio_content", None) if isinstance(existing, str) and existing.strip(): return existing - audio_bytes = getattr(response, "audio_bytes", None) + audio_bytes = ( + _speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config) + if speech_api + else getattr(response, "audio_bytes", None) + ) if not audio_bytes: return None return convert_audio_bytes_to_text( @@ -1005,7 +1035,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], _assert_pcm_int16_speech_hnr( response.audio_bytes, min_hnr_db=min_hnr_db, - sr=int(request_config.get("expected_sample_rate") or _PCM_SPEECH_SAMPLE_RATE_HZ), + sr=_speech_pcm_sample_rate(request_config), ) transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=True) @@ -1016,7 +1046,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], print(f"input text is: {expected_text}") _assert_transcript_matches( transcript, - getattr(response, "audio_bytes", None), + _speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config), expected_text, threshold=0.9, escalation_model=request_config.get("transcript_escalation_model"), diff --git a/tests/helpers/client.py b/tests/helpers/client.py index e9b523a43..5991ab09d 100644 --- a/tests/helpers/client.py +++ b/tests/helpers/client.py @@ -1445,6 +1445,8 @@ class OnlineOmniClient: - min_audio_bytes: optional minimum ``len(audio_bytes)`` checked in ``assert_audio_speech_response`` - transcript_expected_text: local expected spoken text; defaults to ``input`` - transcript_model: primary Whisper model for content checks; defaults to ``small`` + - transcript_pcm_sample_rate: local-only opt-in to transcribe mono int16 PCM; + must be the actual output sample rate, also used for HNR - timeout: request timeout in seconds (float, optional, default 120.0) - stream: whether to use streaming API (bool, optional, default False) diff --git a/tests/helpers/tests/test_assertions.py b/tests/helpers/tests/test_assertions.py index 660a7752f..a64cbe3b9 100644 --- a/tests/helpers/tests/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import io +import wave from types import SimpleNamespace +import numpy as np import pytest from tests.helpers import assertions @@ -149,6 +152,173 @@ def test_escalated_transcript_keeps_declared_language(monkeypatch): assert captured["language"] == "en" +def _pcm_sine(*, sample_rate: int = 24_000, frequency_hz: float = 220.0, duration_s: float = 0.5) -> bytes: + samples = np.arange(round(sample_rate * duration_s), dtype=np.float32) / sample_rate + waveform = 0.35 * np.sin(2 * np.pi * frequency_hz * samples) + return np.rint(waveform * np.iinfo(np.int16).max).astype(" None: + with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file: + assert wav_file.getnchannels() == 1 + assert wav_file.getsampwidth() == 2 + assert wav_file.getframerate() == sample_rate + assert wav_file.readframes(wav_file.getnframes()) == pcm_bytes + + +@pytest.mark.parametrize("run_level", ["advanced_model", "full_model"]) +@pytest.mark.parametrize("sample_rate", [16_000, 24_000]) +def test_pcm_transcript_opt_in_wraps_raw_int16_as_declared_rate_wav(monkeypatch, capsys, run_level, sample_rate): + pcm_bytes = _pcm_sine(sample_rate=sample_rate) + transcriptions: list[tuple[bytes, str, str | None]] = [] + + def transcribe(audio_bytes, model_size="small", language=None): + transcriptions.append((audio_bytes, model_size, language)) + return "hello from raw pcm" + + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe) + + assert_audio_speech_response( + OmniResponse(success=True, audio_bytes=pcm_bytes, audio_format="audio/pcm"), + { + "input": "hello from raw pcm", + "response_format": "pcm", + "transcript_pcm_sample_rate": sample_rate, + }, + run_level, + ) + + assert len(transcriptions) == 1 + wav_bytes, model_size, language = transcriptions[0] + assert model_size == "small" + assert language is None + _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, sample_rate) + assert f"sr={sample_rate}" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("run_level", "request_config"), + [ + ("core_model", {"input": "hello", "response_format": "pcm", "transcript_pcm_sample_rate": 24_000}), + ("advanced_model", {"input": "hello", "response_format": "pcm"}), + ("full_model", {"input": "hello", "response_format": "pcm"}), + ], +) +def test_pcm_asr_remains_disabled_without_opt_in_or_at_core_level(monkeypatch, run_level, request_config): + monkeypatch.setattr( + assertions, + "convert_audio_bytes_to_text", + lambda *_args, **_kwargs: pytest.fail("PCM ASR should not run for this request"), + ) + + assert_audio_speech_response( + OmniResponse(success=True, audio_bytes=_pcm_sine(), audio_format="audio/pcm"), + request_config, + run_level, + ) + + +def test_pcm_transcript_escalation_uses_same_valid_wav_and_rejects_two_wrong_transcripts(monkeypatch): + pcm_bytes = _pcm_sine() + transcriptions: list[tuple[bytes, str, str | None]] = [] + + def transcribe(audio_bytes, model_size="small", language=None): + transcriptions.append((audio_bytes, model_size, language)) + return "this is unrelated speech" + + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe) + + with pytest.raises(AssertionError, match="after ASR escalation"): + assert_audio_speech_response( + OmniResponse(success=True, audio_bytes=pcm_bytes, audio_format="audio/pcm"), + { + "input": "please transcribe this requested sentence", + "response_format": "pcm", + "transcript_pcm_sample_rate": 24_000, + "transcript_escalation_model": "large-v3", + "transcript_language": "en", + }, + "full_model", + ) + + assert [(model_size, language) for _, model_size, language in transcriptions] == [ + ("small", "en"), + ("large-v3", "en"), + ] + for wav_bytes, _, _ in transcriptions: + _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, 24_000) + + +@pytest.mark.parametrize( + "pcm_bytes", + [ + pytest.param(b"\x00\x00" * 12_000, id="silence"), + pytest.param(np.full(12_000, 3_000, dtype="