[CI/Build] Let audio assertions pin the transcript language (#5646)
Signed-off-by: Shenglei Fu <sfu@confluent.io>
This commit is contained in:
1 parent
cd5daaea6a
commit
9d67f8e02f
5 files changed
+165
-9
No files matched your search
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.assertions import _assert_transcript_matches
|
||||
from tests.helpers import assertions
|
||||
from tests.helpers.assertions import _assert_transcript_matches, _resolve_audio_transcript
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -25,3 +28,63 @@ def test_short_transcript_unrelated_text_still_fails():
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
)
|
||||
|
||||
|
||||
def _capture_transcribe(monkeypatch) -> dict:
|
||||
captured: dict = {}
|
||||
|
||||
def fake_convert(raw_bytes, model_size="small", language=None):
|
||||
captured["model_size"] = model_size
|
||||
captured["language"] = language
|
||||
return "London"
|
||||
|
||||
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", fake_convert)
|
||||
return captured
|
||||
|
||||
|
||||
def test_resolve_transcript_honors_declared_language(monkeypatch):
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
|
||||
|
||||
_resolve_audio_transcript(
|
||||
response,
|
||||
{"modalities": ["text", "audio"], "transcript_language": "en"},
|
||||
"full_model",
|
||||
speech_api=False,
|
||||
)
|
||||
|
||||
assert captured["language"] == "en"
|
||||
|
||||
|
||||
def test_resolve_transcript_leaves_language_unset_by_default(monkeypatch):
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
response = SimpleNamespace(audio_content=None, audio_bytes=b"fake-wav")
|
||||
|
||||
_resolve_audio_transcript(
|
||||
response,
|
||||
{"modalities": ["text", "audio"]},
|
||||
"full_model",
|
||||
speech_api=False,
|
||||
)
|
||||
|
||||
assert captured["language"] is None
|
||||
assert captured["model_size"] == "small"
|
||||
|
||||
|
||||
def test_escalated_transcript_keeps_declared_language(monkeypatch):
|
||||
# The escalated pass must honour the same language, otherwise a request that
|
||||
# pins one silently falls back to auto-detection on retry.
|
||||
captured = _capture_transcribe(monkeypatch)
|
||||
|
||||
with pytest.raises(AssertionError, match="after ASR escalation"):
|
||||
_assert_transcript_matches(
|
||||
"totally different words",
|
||||
audio_bytes=b"fake-wav",
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
escalation_model="large-v3",
|
||||
language="en",
|
||||
)
|
||||
|
||||
assert captured["model_size"] == "large-v3"
|
||||
assert captured["language"] == "en"
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
from tests.helpers import media
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _stub_transcribe(monkeypatch) -> dict:
|
||||
"""Capture the kwargs the helper hands to whisper, with no model and no device."""
|
||||
captured: dict = {}
|
||||
|
||||
class FakeModel:
|
||||
def transcribe(self, path, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"text": "London"}
|
||||
|
||||
monkeypatch.setitem(sys.modules, "whisper", SimpleNamespace(load_model=lambda size, device=None: FakeModel()))
|
||||
# Keep the unit test off the accelerator probe so it stays hermetic.
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"vllm_omni.platforms",
|
||||
SimpleNamespace(current_omni_platform=SimpleNamespace(is_available=lambda: False)),
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
def test_transcribe_forwards_requested_language(monkeypatch):
|
||||
captured = _stub_transcribe(monkeypatch)
|
||||
|
||||
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small", language="en")
|
||||
|
||||
assert captured["language"] == "en"
|
||||
|
||||
|
||||
def test_transcribe_defaults_to_auto_language(monkeypatch):
|
||||
# Auto-detect must stay the default: forcing a language globally would break
|
||||
# the non-English audio tests (e.g. the Chinese Qwen3-Omni prompts).
|
||||
captured = _stub_transcribe(monkeypatch)
|
||||
|
||||
media._whisper_transcribe_in_current_process("/tmp/does-not-matter.wav", "small")
|
||||
|
||||
assert captured.get("language") is None
|
||||
|
||||
|
||||
def test_bytes_entrypoint_forwards_language_to_subprocess(monkeypatch, tmp_path):
|
||||
"""Cover the two hops the tests above skip: bytes -> file -> executor.submit.
|
||||
|
||||
The real call crosses a spawn ProcessPoolExecutor, so capture what gets
|
||||
submitted rather than what the worker eventually does.
|
||||
"""
|
||||
submitted: dict = {}
|
||||
|
||||
class FakeExecutor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def submit(self, fn, *args):
|
||||
submitted["fn"] = fn
|
||||
submitted["args"] = args
|
||||
return SimpleNamespace(result=lambda: "London")
|
||||
|
||||
monkeypatch.setattr(media.concurrent.futures, "ProcessPoolExecutor", lambda *a, **kw: FakeExecutor())
|
||||
|
||||
wav = tmp_path / "clip.wav"
|
||||
sf.write(wav, np.zeros(2400, dtype=np.float32), 24000)
|
||||
|
||||
assert media.convert_audio_bytes_to_text(wav.read_bytes(), "small", "en") == "London"
|
||||
|
||||
assert submitted["fn"] is media._whisper_transcribe_in_current_process
|
||||
assert submitted["args"][1:] == ("small", "en")
|
||||
Reference in new issue
Block a user