From ad7f5f94802f125fdc021aca8aee26625452f9ad Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sun, 4 Oct 2026 17:12:01 +0800 Subject: [PATCH] Use accurate ASR for Qwen3-Omni documentation audio Signed-off-by: tzhouam --- .../online_serving/test_qwen3_omni.py | 4 ++- tests/helpers/tests/test_assertions.py | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/examples/online_serving/test_qwen3_omni.py b/tests/examples/online_serving/test_qwen3_omni.py index 23e2ba184..2fbce9dbb 100644 --- a/tests/examples/online_serving/test_qwen3_omni.py +++ b/tests/examples/online_serving/test_qwen3_omni.py @@ -181,7 +181,9 @@ def test_modality_control_003(omni_server) -> None: # Verify text output same as audio output wav_path = extract_last_audio_saved_path(result) - audio_content = convert_audio_file_to_text(output_path=f"./{wav_path}") + # Whisper-small hallucinates a repeated-letter tail on this fixture's WAV. + # Use stronger ASR while retaining the keyword and similarity assertions. + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path}", model_size="large-v3") print(f"text content is: {text_content}") assert "cherry blossom" in audio_content, "The output does not contain any of the keywords." print(f"audio content is: {audio_content}") diff --git a/tests/helpers/tests/test_assertions.py b/tests/helpers/tests/test_assertions.py index e2060152f..d597f06ae 100644 --- a/tests/helpers/tests/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -19,6 +19,36 @@ from tests.helpers.client import OmniResponse pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +@pytest.mark.parametrize("result", ["matching", "wrong_content", "missing_keyword"]) +def test_qwen3_omni_documentation_strong_asr_preserves_content_gates(monkeypatch, result): + from tests.examples.online_serving import test_qwen3_omni as documentation + + expected = "cherry blossom trees fill the foreground while a tower stands against the blue sky." + transcripts = { + "matching": expected, + "wrong_content": "cherry blossom is unrelated to this recording about baking bread in a kitchen.", + "missing_keyword": expected.replace("cherry blossom", "plum flowers"), + } + monkeypatch.setattr( + documentation, + "run_cmd", + lambda _command: f"Chat completion output from text: {expected}\nAudio saved to retained.wav\n", + ) + + def transcribe(output_path, model_size="small", language=None): + assert output_path == "./retained.wav" + return transcripts[result] if model_size == "large-v3" else "f" * 700 + + monkeypatch.setattr(documentation, "convert_audio_file_to_text", transcribe) + error = { + "wrong_content": "The audio content is not same as the text", + "missing_keyword": "The output does not contain any of the keywords", + }.get(result) + expectation = pytest.raises(AssertionError, match=error) if error else nullcontext() + with expectation: + documentation.test_modality_control_003(SimpleNamespace(model="Qwen/Qwen3-Omni-30B-A3B-Instruct")) + + def test_short_transcript_repeat_passes_containment_fallback(): _assert_transcript_matches( " How... how are you?",