[CI/Build] Let audio assertions pin the transcript language (#5646)

Signed-off-by: Shenglei Fu <sfu@confluent.io>

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -468,6 +468,10 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
         "messages": messages,
         "stream": True,
         "key_words": {"text": ["london"]},
+        # A one-word English clip is too short for whisper's language detection to
+        # be stable; unpinned it renders "London" as 런던 / ランデ / 梁敦 and the
+        # containment check then fails on correct audio.
+        "transcript_language": "en",
     }
 
     # Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py).

tests/helpers/assertions.py

@@ -527,7 +527,7 @@ def _resolve_audio_transcript(
     audio_bytes = getattr(response, "audio_bytes", None)
     if not audio_bytes:
         return None
-    return convert_audio_bytes_to_text(audio_bytes)
+    return convert_audio_bytes_to_text(audio_bytes, language=request_config.get("transcript_language"))
 
 
 def assert_omni_response(response: Any, request_config: dict[str, Any], run_level):
@@ -632,6 +632,7 @@ def _assert_transcript_matches(
     *,
     threshold: float,
     escalation_model: str | None = None,
+    language: str | None = None,
 ) -> None:
     """Assert spoken audio matches ``expected_text``.
 
@@ -647,7 +648,8 @@ def _assert_transcript_matches(
     clip is re-transcribed with that stronger ASR and the assertion is decided on
     its verdict -- so a weak whisper-``small`` mishear on a short clip does not
     flake the gate, while a genuine model artifact still fails (the strong ASR
-    mismatches too).
+    mismatches too). ``language`` applies to that escalated pass too, so a request
+    that pins a language does not silently fall back to auto-detection on retry.
     """
     expected = str(expected_text).strip().lower()
     similarity = cosine_similarity_text(transcript.strip().lower(), expected)
@@ -664,7 +666,7 @@ def _assert_transcript_matches(
             f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
             f"escalating to whisper-{escalation_model} to rule out an ASR mishear"
         )
-        strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model)
+        strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model, language=language)
         strong_similarity = cosine_similarity_text(strong_transcript.strip().lower(), expected)
         print(
             f"audio content (whisper-{escalation_model}): {strong_transcript}\n"
@@ -741,6 +743,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
                     expected_text,
                     threshold=0.9,
                     escalation_model=request_config.get("transcript_escalation_model"),
+                    language=request_config.get("transcript_language"),
                 )
         _assert_preset_voice_gender_from_audio(
             response.audio_bytes,

tests/helpers/media.py

@@ -652,7 +652,9 @@ def _serialize_whisper_model_download(model_size: str = "small"):
         f.close()
 
 
-def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "small") -> str:
+def _whisper_transcribe_in_current_process(
+    output_path: str, model_size: str = "small", language: str | None = None
+) -> str:
     import whisper
 
     device_index = None
@@ -684,6 +686,9 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
             temperature=0.0,
             word_timestamps=True,
             condition_on_previous_text=False,
+            # None keeps whisper's auto-detection. Do not default this to a
+            # language: callers include non-English audio tests.
+            language=language,
         )["text"]
     finally:
         del model
@@ -694,15 +699,15 @@ def _whisper_transcribe_in_current_process(output_path: str, model_size: str = "
     return text or ""
 
 
-def convert_audio_file_to_text(output_path: str, model_size: str = "small") -> str:
+def convert_audio_file_to_text(output_path: str, model_size: str = "small", language: str | None = None) -> str:
     """Convert an audio file to text in an isolated subprocess."""
     ctx = multiprocessing.get_context("spawn")
     with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor:
-        future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size)
+        future = executor.submit(_whisper_transcribe_in_current_process, output_path, model_size, language)
         return future.result()
 
 
-def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") -> str:
+def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small", language: str | None = None) -> str:
     output_fd, output_path = tempfile.mkstemp(prefix="test_", suffix=".wav")
     os.close(output_fd)
     if os.environ.get("VLLM_OMNI_KEEP_REQUEST_MEDIA", "").lower() not in ("1", "true", "yes"):
@@ -710,7 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") ->
     data, samplerate = sf.read(io.BytesIO(raw_bytes))
     sf.write(output_path, data, samplerate, format="WAV", subtype="PCM_16")
     print(f"audio data is saved: {output_path}")
-    return convert_audio_file_to_text(output_path, model_size)
+    return convert_audio_file_to_text(output_path, model_size, language)
 
 
 __all__ = [

tests/helpers/tests/test_assertions.py

@@ -1,9 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+from types import SimpleNamespace
+
 import pytest
 
-from tests.helpers.assertions import _assert_transcript_matches
+from tests.helpers import assertions
+from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
@@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails():
             expected_text="how are you",
             threshold=0.9,
         )
+
+
+def _capture_transcribe(monkeypatch) -> dict:
+    captured: dict = {}
+
+    def fake_convert(raw_bytes, model_size="small", language=None):
+        captured["model_size"] = model_size
+        captured["language"] = language
+        return "London"
+
+    monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert)
+    return captured
+
+
+def test_resolve_transcript_honors_declared_language(monkeypatch):
+    captured = _capture_transcribe(monkeypatch)
+    response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
+
+    _resolve_audio_transcript(
+        response,
+        {"modalities": ["text", "audio"], "transcript_language": "en"},
+        "full_model",
+        speech_api=False,
+    )
+
+    assert captured["language"] == "en"
+
+
+def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch):
+    captured = _capture_transcribe(monkeypatch)
+    response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
+
+    _resolve_audio_transcript(
+        response,
+        {"modalities": ["text", "audio"]},
+        "full_model",
+        speech_api=False,
+    )
+
+    assert captured["language"] is None
+    assert captured["model_size"] == "small"
+
+
+def test_escalated_transcript_keeps_declared_language(monkeypatch):
+    # The escalated pass must honour the same language, otherwise a request that
+    # pins one silently falls back to auto-detection on retry.
+    captured = _capture_transcribe(monkeypatch)
+
+    with pytest.raises(AssertionError, match="after ASR escalation"):
+        _assert_transcript_matches(
+            "totally different words",
+            audio_bytes=b"fake-wav",
+            expected_text="how are you",
+            threshold=0.9,
+            escalation_model="large-v3",
+            language="en",
+        )
+
+    assert captured["model_size"] == "large-v3"
+    assert captured["language"] == "en"

tests/helpers/tests/test_media.py

@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import sys
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import soundfile as sf
+
+from tests.helpers import media
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def _stub_transcribe(monkeypatch) -> dict:
+    """Capture the kwargs the helper hands to whisper, with no model and no device."""
+    captured: dict = {}
+
+    class FakeModel:
+        def transcribe(self, path, **kwargs):
+            captured.update(kwargs)
+            return {"text": "London"}
+
+    monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel()))
+    # Keep the unit test off the accelerator probe so it stays hermetic.
+    monkeypatch.setitem(
+        sys.modules,
+        "vllm_omni.platforms",
+        SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)),
+    )
+    return captured
+
+
+def test_transcribe_forwards_requested_language(monkeypatch):
+    captured = _stub_transcribe(monkeypatch)
+
+    media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en")
+
+    assert captured["language"] == "en"
+
+
+def test_transcribe_defaults_to_auto_language(monkeypatch):
+    # Auto-detect must stay the default: forcing a language globally would break
+    # the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts).
+    captured = _stub_transcribe(monkeypatch)
+
+    media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small")
+
+    assert captured.get("language") is None
+
+
+def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path):
+    """Cover the two hops the tests above skip: bytes -> file -> executor.submit.
+
+    The real call crosses a spawn ProcessPoolExecutor, so capture what gets
+    submitted rather than what the worker eventually does.
+    """
+    submitted: dict = {}
+
+    class FakeExecutor:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def submit(self, fn, *args):
+            submitted["fn"] = fn
+            submitted["args"] = args
+            return SimpleNamespace(result=lambda: "London")
+
+    monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor())
+
+    wav = tmp_path / "clip.wav"
+    sf.write(wav, np.zeros(2400, dtype=np.float32), 24000)
+
+    assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London"
+
+    assert submitted["fn"] is media._whisper_transcribe_in_current_process
+    assert submitted["args"][1:] == ("small", "en")