[Bugfix] Validate Breeze PCM speech content and calibrate HNR guard (#8044)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>

tests/e2e/online_serving/test_breeze_tts_2.py

@@ -52,14 +52,16 @@ def test_english_streaming(omni_server, online_client, run_level) -> None:
             "input": "Hello, this is a demonstration of natural speech synthesis.",
             "instructions": "A warm, clear female voice speaking English at a moderate pace.",
             "transcript_language": "en",
+            "transcript_pcm_sample_rate": 24000,
+            "transcript_escalation_model": "large-v3",
             "response_format": "pcm",
             "sample_rate": 24000,
             "min_audio_bytes": 24000,
             "stream": True,
             "stream_format": "audio",
-            # The reference eager runtime scores 0.56 dB on this same prompt
-            # (seed 42, CFG=1); the shared 1 dB floor rejects it as well.
-            "min_hnr_db": 0.0,
+            # This fixed prompt scores -0.11 dB on L4 and 0.56 dB on H100.
+            # Keep a small HNR margin and also verify the spoken words above.
+            "min_hnr_db": -1.0,
             "seed": 42,
             "max_new_tokens": 8 if run_level == "core_model" else 250,
         }

tests/helpers/assertions.py

@@ -688,7 +688,9 @@ def _compute_pcm_hnr_db(pcm_samples: np.ndarray, sr: int = _PCM_SPEECH_SAMPLE_RA
         peak = float(np.max(ac[min_lag:max_lag]))
         if 0 < peak < 1:
             hnr_values.append(10 * np.log10(peak / (1 - peak + 1e-10)))
-    return float(np.mean(hnr_values)) if hnr_values else 0.0
+    # Silence and constant signals provide no usable harmonic estimate. They
+    # must not pass a model-specific floor of zero or below.
+    return float(np.mean(hnr_values)) if hnr_values else float("-inf")
 
 
 def _assert_pcm_int16_speech_hnr(
@@ -714,10 +716,7 @@ def _assert_pcm_int16_speech_hnr(
     pcm_samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
     hnr = _compute_pcm_hnr_db(pcm_samples, sr=sr)
     print(f"PCM speech HNR: {hnr:.2f} dB (threshold: {min_hnr_db} dB, sr={sr})")
-    assert hnr >= min_hnr_db, (
-        f"Audio distortion detected: HNR={hnr:.2f} dB < {min_hnr_db} dB. "
-        "Voice clone decoder may be losing ref_code speaker context on later chunks."
-    )
+    assert hnr >= min_hnr_db, f"PCM speech HNR={hnr:.2f} dB is below the configured floor of {min_hnr_db} dB."
 
 
 def _response_has_audio_output(response: Any) -> bool:
@@ -754,11 +753,38 @@ def _omni_assertion_needs_audio_transcript(request_config: dict[str, Any], run_l
 def _speech_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str | None) -> bool:
     if run_level not in {"advanced_model", "full_model"}:
         return False
-    if request_config.get("response_format") == "pcm":
+    if request_config.get("response_format") == "pcm" and "transcript_pcm_sample_rate" not in request_config:
         return False
     return bool(request_config.get("transcript_expected_text", request_config.get("input")))
 
 
+def _speech_pcm_sample_rate(request_config: dict[str, Any]) -> int:
+    expected = request_config.get("expected_sample_rate")
+    if "transcript_pcm_sample_rate" in request_config:
+        sample_rate = request_config["transcript_pcm_sample_rate"]
+        assert type(sample_rate) is int and sample_rate > 0, "transcript_pcm_sample_rate must be a positive integer"
+        assert expected is None or int(expected) == sample_rate, (
+            "transcript_pcm_sample_rate must match expected_sample_rate"
+        )
+        return sample_rate
+    return int(expected or _PCM_SPEECH_SAMPLE_RATE_HZ)
+
+
+def _speech_audio_for_transcription(audio_bytes: bytes | None, request_config: dict[str, Any]) -> bytes | None:
+    """Give ASR a WAV container when a speech test opts into raw PCM transcription."""
+    if not audio_bytes or request_config.get("response_format") != "pcm":
+        return audio_bytes
+    sample_rate = _speech_pcm_sample_rate(request_config)
+    assert len(audio_bytes) % 2 == 0, "PCM byte length must be aligned to int16"
+    with io.BytesIO() as buffer:
+        with wave.open(buffer, "wb") as wav_file:
+            wav_file.setnchannels(1)
+            wav_file.setsampwidth(2)
+            wav_file.setframerate(sample_rate)
+            wav_file.writeframes(audio_bytes)
+        return buffer.getvalue()
+
+
 def _resolve_audio_transcript(
     response: Any,
     request_config: dict[str, Any],
@@ -777,7 +803,11 @@ def _resolve_audio_transcript(
     existing = getattr(response, "audio_content", None)
     if isinstance(existing, str) and existing.strip():
         return existing
-    audio_bytes = getattr(response, "audio_bytes", None)
+    audio_bytes = (
+        _speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config)
+        if speech_api
+        else getattr(response, "audio_bytes", None)
+    )
     if not audio_bytes:
         return None
     return convert_audio_bytes_to_text(
@@ -1005,7 +1035,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
             _assert_pcm_int16_speech_hnr(
                 response.audio_bytes,
                 min_hnr_db=min_hnr_db,
-                sr=int(request_config.get("expected_sample_rate") or _PCM_SPEECH_SAMPLE_RATE_HZ),
+                sr=_speech_pcm_sample_rate(request_config),
             )
 
         transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=True)
@@ -1016,7 +1046,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
                 print(f"input text is: {expected_text}")
                 _assert_transcript_matches(
                     transcript,
-                    getattr(response, "audio_bytes", None),
+                    _speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config),
                     expected_text,
                     threshold=0.9,
                     escalation_model=request_config.get("transcript_escalation_model"),

tests/helpers/client.py

@@ -1445,6 +1445,8 @@ class OnlineOmniClient:
           - min_audio_bytes: optional minimum ``len(audio_bytes)`` checked in ``assert_audio_speech_response``
           - transcript_expected_text: local expected spoken text; defaults to ``input``
           - transcript_model: primary Whisper model for content checks; defaults to ``small``
+          - transcript_pcm_sample_rate: local-only opt-in to transcribe mono int16 PCM;
+            must be the actual output sample rate, also used for HNR
           - timeout: request timeout in seconds (float, optional, default 120.0)
           - stream: whether to use streaming API (bool, optional, default False)
 

tests/helpers/tests/test_assertions.py

@@ -1,8 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+import io
+import wave
 from types import SimpleNamespace
 
+import numpy as np
 import pytest
 
 from tests.helpers import assertions
@@ -149,6 +152,173 @@ def test_escalated_transcript_keeps_declared_language(monkeypatch):
     assert captured["language"] == "en"
 
 
+def _pcm_sine(*, sample_rate: int = 24_000, frequency_hz: float = 220.0, duration_s: float = 0.5) -> bytes:
+    samples = np.arange(round(sample_rate * duration_s), dtype=np.float32) / sample_rate
+    waveform = 0.35 * np.sin(2 * np.pi * frequency_hz * samples)
+    return np.rint(waveform * np.iinfo(np.int16).max).astype("<i2").tobytes()
+
+
+def _assert_wav_wraps_pcm(wav_bytes: bytes, pcm_bytes: bytes, sample_rate: int) -> None:
+    with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
+        assert wav_file.getnchannels() == 1
+        assert wav_file.getsampwidth() == 2
+        assert wav_file.getframerate() == sample_rate
+        assert wav_file.readframes(wav_file.getnframes()) == pcm_bytes
+
+
+@pytest.mark.parametrize("run_level", ["advanced_model", "full_model"])
+@pytest.mark.parametrize("sample_rate", [16_000, 24_000])
+def test_pcm_transcript_opt_in_wraps_raw_int16_as_declared_rate_wav(monkeypatch, capsys, run_level, sample_rate):
+    pcm_bytes = _pcm_sine(sample_rate=sample_rate)
+    transcriptions: list[tuple[bytes, str, str | None]] = []
+
+    def transcribe(audio_bytes, model_size="small", language=None):
+        transcriptions.append((audio_bytes, model_size, language))
+        return "hello from raw pcm"
+
+    monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe)
+
+    assert_audio_speech_response(
+        OmniResponse(success=True, audio_bytes=pcm_bytes, audio_format="audio/pcm"),
+        {
+            "input": "hello from raw pcm",
+            "response_format": "pcm",
+            "transcript_pcm_sample_rate": sample_rate,
+        },
+        run_level,
+    )
+
+    assert len(transcriptions) == 1
+    wav_bytes, model_size, language = transcriptions[0]
+    assert model_size == "small"
+    assert language is None
+    _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, sample_rate)
+    assert f"sr={sample_rate}" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    ("run_level", "request_config"),
+    [
+        ("core_model", {"input": "hello", "response_format": "pcm", "transcript_pcm_sample_rate": 24_000}),
+        ("advanced_model", {"input": "hello", "response_format": "pcm"}),
+        ("full_model", {"input": "hello", "response_format": "pcm"}),
+    ],
+)
+def test_pcm_asr_remains_disabled_without_opt_in_or_at_core_level(monkeypatch, run_level, request_config):
+    monkeypatch.setattr(
+        assertions,
+        "convert_audio_bytes_to_text",
+        lambda *_args, **_kwargs: pytest.fail("PCM ASR should not run for this request"),
+    )
+
+    assert_audio_speech_response(
+        OmniResponse(success=True, audio_bytes=_pcm_sine(), audio_format="audio/pcm"),
+        request_config,
+        run_level,
+    )
+
+
+def test_pcm_transcript_escalation_uses_same_valid_wav_and_rejects_two_wrong_transcripts(monkeypatch):
+    pcm_bytes = _pcm_sine()
+    transcriptions: list[tuple[bytes, str, str | None]] = []
+
+    def transcribe(audio_bytes, model_size="small", language=None):
+        transcriptions.append((audio_bytes, model_size, language))
+        return "this is unrelated speech"
+
+    monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe)
+
+    with pytest.raises(AssertionError, match="after ASR escalation"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=pcm_bytes, audio_format="audio/pcm"),
+            {
+                "input": "please transcribe this requested sentence",
+                "response_format": "pcm",
+                "transcript_pcm_sample_rate": 24_000,
+                "transcript_escalation_model": "large-v3",
+                "transcript_language": "en",
+            },
+            "full_model",
+        )
+
+    assert [(model_size, language) for _, model_size, language in transcriptions] == [
+        ("small", "en"),
+        ("large-v3", "en"),
+    ]
+    for wav_bytes, _, _ in transcriptions:
+        _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, 24_000)
+
+
+@pytest.mark.parametrize(
+    "pcm_bytes",
+    [
+        pytest.param(b"\x00\x00" * 12_000, id="silence"),
+        pytest.param(np.full(12_000, 3_000, dtype="<i2").tobytes(), id="constant_signal"),
+    ],
+)
+@pytest.mark.parametrize("min_hnr_db", [0, -1, -5])
+def test_pcm_unvoiced_audio_fails_even_when_the_hnr_floor_is_lowered(pcm_bytes, min_hnr_db):
+    with pytest.raises(AssertionError, match="HNR"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=pcm_bytes, audio_format="audio/pcm"),
+            {"response_format": "pcm", "min_hnr_db": min_hnr_db},
+            "advanced_model",
+        )
+
+
+@pytest.mark.parametrize("min_hnr_db", [1.0, -1.0])
+def test_pcm_white_noise_fails_the_speech_hnr_gate(min_hnr_db):
+    rng = np.random.default_rng(0)
+    noise = rng.integers(np.iinfo(np.int16).min, np.iinfo(np.int16).max, 24_000, dtype=np.int16).astype("<i2")
+
+    with pytest.raises(AssertionError, match="HNR"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=noise.tobytes(), audio_format="audio/pcm"),
+            {"response_format": "pcm", "min_hnr_db": min_hnr_db},
+            "advanced_model",
+        )
+
+
+def test_pcm_voiced_signal_passes_the_default_speech_hnr_gate():
+    assert_audio_speech_response(
+        OmniResponse(success=True, audio_bytes=_pcm_sine(), audio_format="audio/pcm"),
+        {"response_format": "pcm"},
+        "advanced_model",
+    )
+
+
+@pytest.mark.parametrize("sample_rate", [None, 0, -1, True, 24000.5, "not-a-rate"])
+def test_pcm_transcript_opt_in_rejects_invalid_declared_sample_rate(sample_rate):
+    with pytest.raises(AssertionError, match="transcript_pcm_sample_rate"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=_pcm_sine(), audio_format="audio/pcm"),
+            {
+                "input": "hello",
+                "response_format": "pcm",
+                "transcript_pcm_sample_rate": sample_rate,
+            },
+            "advanced_model",
+        )
+
+
+def test_pcm_transcript_and_hnr_rates_must_agree():
+    with pytest.raises(AssertionError, match="transcript_pcm_sample_rate must match expected_sample_rate"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=_pcm_sine(), audio_format="audio/pcm"),
+            {"response_format": "pcm", "transcript_pcm_sample_rate": 24_000, "expected_sample_rate": 16_000},
+            "full_model",
+        )
+
+
+def test_pcm_rejects_odd_length_int16_payload():
+    with pytest.raises(AssertionError, match="aligned to int16"):
+        assert_audio_speech_response(
+            OmniResponse(success=True, audio_bytes=b"\x00\x01\x02", audio_format="audio/pcm"),
+            {"response_format": "pcm"},
+            "advanced_model",
+        )
+
+
 def test_bounded_tail_after_complete_answer_passes():
     # The exact shape from the #6815 nightly failure: the full answer is spoken
     # verbatim, then a two-word unrelated tail trips the cosine gate.