diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 80eb5fce3..86e794c7a 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -158,6 +158,11 @@ steps: - label: ":full_moon: TTS ยท Function Test with L4" source_file_dependencies: tts_qwen3_tts_function timeout_in_minutes: 120 + env: + VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR: . + artifact_paths: + - failed_speech_*.wav + - failed_speech_*.json commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index f6559e0c3..7694761c6 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -4,9 +4,11 @@ """Assertion and response validation helpers for tests.""" import base64 +import hashlib import io import json import math +import os import tempfile import threading import wave @@ -999,6 +1001,29 @@ def _assert_transcript_matches( ) +def _retain_failed_speech_audio(audio_bytes: bytes | None, transcript: str, expected_text: Any, error: AssertionError) -> None: + """Retain the exact ASR input when a CI speech-content assertion fails.""" + directory = os.environ.get("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR") + if not directory or not audio_bytes: + return + stem = f"failed_speech_{hashlib.sha256(audio_bytes).hexdigest()[:16]}" + root = Path(directory) + try: + root.mkdir(parents=True, exist_ok=True) + (root / f"{stem}.wav").write_bytes(audio_bytes) + (root / f"{stem}.json").write_text( + json.dumps( + {"transcript": transcript, "expected_text": str(expected_text), "assertion": str(error)}, + ensure_ascii=False, + indent=2, + ) + ) + print(f"Retained failed speech ASR input: {root / f'{stem}.wav'}") + except OSError as exc: + # Diagnostic I/O must never replace or suppress the content failure. + print(f"Could not retain failed speech ASR input: {exc}") + + def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str | None = None) -> None: """Validate speech API results from :class:`~tests.helpers.client.OmniResponse`. @@ -1063,14 +1088,21 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], if expected_text: print(f"audio content is: {transcript}") print(f"input text is: {expected_text}") - _assert_transcript_matches( - transcript, - _speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config), - expected_text, - threshold=0.9, - escalation_model=request_config.get("transcript_escalation_model"), - language=request_config.get("transcript_language"), + transcription_audio = _speech_audio_for_transcription( + getattr(response, "audio_bytes", None), request_config ) + try: + _assert_transcript_matches( + transcript, + transcription_audio, + expected_text, + threshold=0.9, + escalation_model=request_config.get("transcript_escalation_model"), + language=request_config.get("transcript_language"), + ) + except AssertionError as exc: + _retain_failed_speech_audio(transcription_audio, transcript, expected_text, exc) + raise _assert_preset_voice_gender_from_audio( response.audio_bytes, request_config.get("voice"), diff --git a/tests/helpers/tests/test_assertions.py b/tests/helpers/tests/test_assertions.py index e2060152f..b4e54d094 100644 --- a/tests/helpers/tests/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import io +import json import wave +from contextlib import nullcontext from types import SimpleNamespace import numpy as np @@ -249,6 +251,87 @@ def test_pcm_transcript_escalation_uses_same_valid_wav_and_rejects_two_wrong_tra _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, 24_000) +@pytest.mark.parametrize("response_format", ["wav", "pcm"]) +@pytest.mark.parametrize("escalation_model", [None, "large-v3"]) +def test_failed_speech_capture_retains_exact_asr_input_and_original_failure( + monkeypatch, tmp_path, response_format, escalation_model +): + pcm_bytes = _pcm_sine() + wav = io.BytesIO() + with wave.open(wav, "wb") as writer: + writer.setnchannels(1) + writer.setsampwidth(2) + writer.setframerate(24_000) + writer.writeframes(pcm_bytes) + audio_bytes = wav.getvalue() if response_format == "wav" else pcm_bytes + transcribed = [] + + def transcribe(audio_bytes, **kwargs): + transcribed.append(audio_bytes) + return "this is unrelated speech" + + directory = tmp_path / "failed_audio" + monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory)) + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe) + config = { + "input": "please read this requested sentence accurately", + "response_format": response_format, + "transcript_pcm_sample_rate": 24_000, + "transcript_escalation_model": escalation_model, + } + response = OmniResponse(success=True, audio_bytes=audio_bytes, audio_format=f"audio/{response_format}") + with pytest.raises(AssertionError, match="Transcript doesn't match input") as error: + assert_audio_speech_response(response, config, "full_model") + + clips = list(directory.glob("*.wav")) + assert len(clips) == 1 + assert all(clip == clips[0].read_bytes() for clip in transcribed) + _assert_wav_wraps_pcm(clips[0].read_bytes(), pcm_bytes, 24_000) + metadata = json.loads(clips[0].with_suffix(".json").read_text()) + assert metadata["expected_text"] == config["input"] + assert metadata["transcript"] == "this is unrelated speech" + assert metadata["assertion"] == str(error.value) + + +@pytest.mark.parametrize("capture_enabled", [False, True]) +@pytest.mark.parametrize("content_failure", [False, True]) +def test_speech_capture_is_opt_in_and_does_not_write_successes( + monkeypatch, tmp_path, capture_enabled, content_failure +): + directory = tmp_path / "failed_audio" + if capture_enabled: + monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory)) + else: + monkeypatch.delenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", raising=False) + expected = "please read this requested sentence accurately" + monkeypatch.setattr( + assertions, + "convert_audio_bytes_to_text", + lambda *_args, **_kwargs: "this is unrelated speech" if content_failure else expected, + ) + with pytest.raises(AssertionError, match="Transcript doesn't match input") if content_failure else nullcontext(): + assert_audio_speech_response( + OmniResponse(success=True, audio_bytes=b"audio", audio_format="audio/wav"), + {"input": expected, "response_format": "wav"}, + "full_model", + ) + assert directory.exists() is (capture_enabled and content_failure) + assert len(list(tmp_path.rglob("*.wav"))) == int(capture_enabled and content_failure) + + +def test_speech_capture_io_failure_does_not_mask_content_failure(monkeypatch, tmp_path): + directory = tmp_path / "blocked" + directory.write_text("a file cannot be an output directory") + monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory)) + monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", lambda *_args, **_kwargs: "this is unrelated speech") + with pytest.raises(AssertionError, match="Transcript doesn't match input"): + assert_audio_speech_response( + OmniResponse(success=True, audio_bytes=b"audio", audio_format="audio/wav"), + {"input": "please read this requested sentence accurately", "response_format": "wav"}, + "full_model", + ) + + @pytest.mark.parametrize( "pcm_bytes", [