[CI/Build] Let audio assertions pin the transcript language (#5646)

Signed-off-by: Shenglei Fu <sfu@confluent.io>
This commit is contained in:
Shenglei Fu authored and GitHub committed 2026-07-31 21:44:28 -07:00
1 parent cd5daaea6a
commit 9d67f8e02f
5 files changed
+165 -9

No files matched your search

+64 -1
View File
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from tests.helpers.assertions import _assert_transcript_matches
from tests.helpers import assertions
from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails():
expected_text="how are you",
threshold=0.9,
)
def _capture_transcribe(monkeypatch) -> dict:
captured: dict = {}
def fake_convert(raw_bytes, model_size="small", language=None):
captured["model_size"] = model_size
captured["language"] = language
return "London"
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert)
return captured
def test_resolve_transcript_honors_declared_language(monkeypatch):
captured = _capture_transcribe(monkeypatch)
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
_resolve_audio_transcript(
response,
{"modalities": ["text", "audio"], "transcript_language": "en"},
"full_model",
speech_api=False,
)
assert captured["language"] == "en"
def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch):
captured = _capture_transcribe(monkeypatch)
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
_resolve_audio_transcript(
response,
{"modalities": ["text", "audio"]},
"full_model",
speech_api=False,
)
assert captured["language"] is None
assert captured["model_size"] == "small"
def test_escalated_transcript_keeps_declared_language(monkeypatch):
# The escalated pass must honour the same language, otherwise a request that
# pins one silently falls back to auto-detection on retry.
captured = _capture_transcribe(monkeypatch)
with pytest.raises(AssertionError, match="after ASR escalation"):
_assert_transcript_matches(
"totally different words",
audio_bytes=b"fake-wav",
expected_text="how are you",
threshold=0.9,
escalation_model="large-v3",
language="en",
)
assert captured["model_size"] == "large-v3"
assert captured["language"] == "en"
+81
View File
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from types import SimpleNamespace
import numpy as np
import pytest
import soundfile as sf
from tests.helpers import media
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _stub_transcribe(monkeypatch) -> dict:
"""Capture the kwargs the helper hands to whisper, with no model and no device."""
captured: dict = {}
class FakeModel:
def transcribe(self, path, **kwargs):
captured.update(kwargs)
return {"text": "London"}
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel()))
# Keep the unit test off the accelerator probe so it stays hermetic.
monkeypatch.setitem(
sys.modules,
"vllm_omni.platforms",
SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)),
)
return captured
def test_transcribe_forwards_requested_language(monkeypatch):
captured = _stub_transcribe(monkeypatch)
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en")
assert captured["language"] == "en"
def test_transcribe_defaults_to_auto_language(monkeypatch):
# Auto-detect must stay the default: forcing a language globally would break
# the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts).
captured = _stub_transcribe(monkeypatch)
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small")
assert captured.get("language") is None
def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path):
"""Cover the two hops the tests above skip: bytes -> file -> executor.submit.
The real call crosses a spawn ProcessPoolExecutor, so capture what gets
submitted rather than what the worker eventually does.
"""
submitted: dict = {}
class FakeExecutor:
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def submit(self, fn, *args):
submitted["fn"] = fn
submitted["args"] = args
return SimpleNamespace(result=lambda: "London")
monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor())
wav = tmp_path / "clip.wav"
sf.write(wav, np.zeros(2400, dtype=np.float32), 24000)
assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London"
assert submitted["fn"] is media._whisper_transcribe_in_current_process
assert submitted["args"][1:] == ("small", "en")