From 9d67f8e02fd4b386a62b8bdac2bed95039ef6753 Mon Sep 17 00:00:00 2001 From: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:44:28 -0700 Subject: [PATCH] [CI/Build] Let audio assertions pin the transcript language (#5646) Signed-off-by: Shenglei Fu --- .../test_qwen3_omni_expansion.py | 4 + tests/helpers/assertions.py | 9 ++- tests/helpers/media.py | 15 ++-- tests/helpers/tests/test_assertions.py | 65 ++++++++++++++- tests/helpers/tests/test_media.py | 81 +++++++++++++++++++ 5 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 tests/helpers/tests/test_media.py diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 3ded285ce..1ba94c011 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -468,6 +468,10 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None: "messages": messages, "stream": True, "key_words": {"text": ["london"]}, + # A one-word English clip is too short for whisper's language detection to + # be stable; unpinned it renders "London" as 런던 / ランデ / 梁敦 and the + # containment check then fails on correct audio. + "transcript_language": "en", } # Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py). diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index 11e86ed8c..9fe34f9b5 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -527,7 +527,7 @@ def _resolve_audio_transcript( audio_bytes = getattr(response, "audio_bytes", None) if not audio_bytes: return None - return convert_audio_bytes_to_text(audio_bytes) + return convert_audio_bytes_to_text(audio_bytes, language=request_config.get("transcript_language")) def assert_omni_response(response: Any, request_config: dict[str, Any], run_level): @@ -632,6 +632,7 @@ def _assert_transcript_matches( *, threshold: float, escalation_model: str | None = None, + language: str | None = None, ) -> None: """Assert spoken audio matches ``expected_text``. @@ -647,7 +648,8 @@ def _assert_transcript_matches( clip is re-transcribed with that stronger ASR and the assertion is decided on its verdict -- so a weak whisper-``small`` mishear on a short clip does not flake the gate, while a genuine model artifact still fails (the strong ASR - mismatches too). + mismatches too). ``language`` applies to that escalated pass too, so a request + that pins a language does not silently fall back to auto-detection on retry. """ expected = str(expected_text).strip().lower() similarity = cosine_similarity_text(transcript.strip().lower(), expected) @@ -664,7 +666,7 @@ def _assert_transcript_matches( f"whisper-small below threshold ({similarity:.2f} <= {threshold}); " f"escalating to whisper-{escalation_model} to rule out an ASR mishear" ) - strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model) + strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model, language=language) strong_similarity = cosine_similarity_text(strong_transcript.strip().lower(), expected) print( f"audio content (whisper-{escalation_model}): {strong_transcript}\n" @@ -741,6 +743,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], expected_text, threshold=0.9, escalation_model=request_config.get("transcript_escalation_model"), + language=request_config.get("transcript_language"), ) _assert_preset_voice_gender_from_audio( response.audio_bytes, diff --git a/tests/helpers/media.py b/tests/helpers/media.py index be444051d..93e136f57 100644 --- a/tests/helpers/media.py +++ b/tests/helpers/media.py @@ -652,7 +652,9 @@ def _serialize_whisper_model_download(model_size: str = "small"): f.close() -def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "small") -> str: +def _whisper_transcribe_in_current_process( + output_path: str, model_size: str = "small", language: str | None = None +) -> str: import whisper device_index = None @@ -684,6 +686,9 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = " temperature=0.0, word_timestamps=True, condition_on_previous_text=False, + # None keeps whisper's auto-detection. Do not default this to a + # language: callers include non-English audio tests. + language=language, )["text"] finally: del model @@ -694,15 +699,15 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = " return text or "" -def convert_audio_file_to_text(output_path: str, model_size: str = "small") -> str: +def convert_audio_file_to_text(output_path: str, model_size: str = "small", language: str | None = None) -> str: """Convert an audio file to text in an isolated subprocess.""" ctx = multiprocessing.get_context("spawn") with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor: - future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size) + future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size, language) return future.result() -def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") -> str: +def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small", language: str | None = None) -> str: output_fd, output_path = tempfile.mkstemp(prefix="test_", suffix=".wav") os.close(output_fd) if os.environ.get("VLLM_OMNI_KEEP_REQUEST_MEDIA", "").lower() not in ("1", "true", "yes"): @@ -710,7 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") -> data, samplerate = sf.read(io.BytesIO(raw_bytes)) sf.write(output_path, data, samplerate, format="WAV", subtype="PCM_16") print(f"audio data is saved: {output_path}") - return convert_audio_file_to_text(output_path, model_size) + return convert_audio_file_to_text(output_path, model_size, language) __all__ = [ diff --git a/tests/helpers/tests/test_assertions.py b/tests/helpers/tests/test_assertions.py index 8293349fa..9cb1560b9 100644 --- a/tests/helpers/tests/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + import pytest -from tests.helpers.assertions import _assert_transcript_matches +from tests.helpers import assertions +from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails(): expected_text="how are you", threshold=0.9, ) + + +def _capture_transcribe(monkeypatch) -> dict: + captured: dict = {} + + def fake_convert(raw_bytes, model_size="small", language=None): + captured["model_size"] = model_size + captured["language"] = language + return "London" + + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert) + return captured + + +def test_resolve_transcript_honors_declared_language(monkeypatch): + captured = _capture_transcribe(monkeypatch) + response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav") + + _resolve_audio_transcript( + response, + {"modalities": ["text", "audio"], "transcript_language": "en"}, + "full_model", + speech_api=False, + ) + + assert captured["language"] == "en" + + +def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch): + captured = _capture_transcribe(monkeypatch) + response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav") + + _resolve_audio_transcript( + response, + {"modalities": ["text", "audio"]}, + "full_model", + speech_api=False, + ) + + assert captured["language"] is None + assert captured["model_size"] == "small" + + +def test_escalated_transcript_keeps_declared_language(monkeypatch): + # The escalated pass must honour the same language, otherwise a request that + # pins one silently falls back to auto-detection on retry. + captured = _capture_transcribe(monkeypatch) + + with pytest.raises(AssertionError, match="after ASR escalation"): + _assert_transcript_matches( + "totally different words", + audio_bytes=b"fake-wav", + expected_text="how are you", + threshold=0.9, + escalation_model="large-v3", + language="en", + ) + + assert captured["model_size"] == "large-v3" + assert captured["language"] == "en" diff --git a/tests/helpers/tests/test_media.py b/tests/helpers/tests/test_media.py new file mode 100644 index 000000000..4a8e5393d --- /dev/null +++ b/tests/helpers/tests/test_media.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import sys +from types import SimpleNamespace + +import numpy as np +import pytest +import soundfile as sf + +from tests.helpers import media + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _stub_transcribe(monkeypatch) -> dict: + """Capture the kwargs the helper hands to whisper, with no model and no device.""" + captured: dict = {} + + class FakeModel: + def transcribe(self, path, **kwargs): + captured.update(kwargs) + return {"text": "London"} + + monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel())) + # Keep the unit test off the accelerator probe so it stays hermetic. + monkeypatch.setitem( + sys.modules, + "vllm_omni.platforms", + SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)), + ) + return captured + + +def test_transcribe_forwards_requested_language(monkeypatch): + captured = _stub_transcribe(monkeypatch) + + media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en") + + assert captured["language"] == "en" + + +def test_transcribe_defaults_to_auto_language(monkeypatch): + # Auto-detect must stay the default: forcing a language globally would break + # the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts). + captured = _stub_transcribe(monkeypatch) + + media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small") + + assert captured.get("language") is None + + +def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path): + """Cover the two hops the tests above skip: bytes -> file -> executor.submit. + + The real call crosses a spawn ProcessPoolExecutor, so capture what gets + submitted rather than what the worker eventually does. + """ + submitted: dict = {} + + class FakeExecutor: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def submit(self, fn, *args): + submitted["fn"] = fn + submitted["args"] = args + return SimpleNamespace(result=lambda: "London") + + monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor()) + + wav = tmp_path / "clip.wav" + sf.write(wav, np.zeros(2400, dtype=np.float32), 24000) + + assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London" + + assert submitted["fn"] is media._whisper_transcribe_in_current_process + assert submitted["args"][1:] == ("small", "en")