diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index b3abea735..11e86ed8c 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -19,19 +19,20 @@ from PIL import Image from tests.helpers.media import ( convert_audio_bytes_to_text, cosine_similarity_text, + preprocess_text, ) _GENDER_PIPELINE = None _GENDER_PIPELINE_LOCK = threading.Lock() # Transcript gates default to whisper ``small`` for speed. ``small`` mishears a # short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a -# leading SFX token), which flakes the deterministic similarity gate. A test can -# opt in to ASR escalation by setting ``transcript_escalation_model`` to a -# whisper model name (e.g. ``"large-v3"``) in its request_config: on a failed -# fast pass the clip is re-transcribed with that stronger ASR before the test -# fails, so a weak-ASR mishear is rescued while a genuine model artifact still -# fails (the strong ASR mismatches too). Tests that do not opt in keep the -# strict behaviour. +# leading SFX token), which flakes the deterministic similarity gate. Short +# clips first get a conservative containment fallback for minor ASR repeats/noise. +# A test can also opt in to ASR escalation by setting +# ``transcript_escalation_model`` to a whisper model name (e.g. ``"large-v3"``) +# in its request_config: on a failed fast pass the clip is re-transcribed with +# that stronger ASR before the test fails, so a weak-ASR mishear is rescued while +# a genuine model artifact still fails (the strong ASR mismatches too). _PCM_SPEECH_SAMPLE_RATE_HZ = 24_000 _MIN_PCM_SPEECH_HNR_DB = 1.0 _PRESET_VOICE_GENDER_MAP: dict[str, str] = { @@ -43,6 +44,23 @@ _PRESET_VOICE_GENDER_MAP: dict[str, str] = { } +def _short_transcript_contains_expected(transcript: str, expected: str) -> bool: + """Allow minor ASR repeats/noise for very short speech clips.""" + transcript_clean = preprocess_text(transcript) + expected_clean = preprocess_text(expected) + if not transcript_clean or not expected_clean: + return False + + transcript_words = transcript_clean.split() + expected_words = expected_clean.split() + if not transcript_words or not expected_words: + return False + + short_text = min(len(transcript_clean), len(expected_clean)) <= 15 + small_word_delta = len(transcript_words) <= len(expected_words) + 2 + return short_text and small_word_delta and expected_clean in transcript_clean + + def assert_image_diffusion_response( response: "DiffusionResponse", request_config: dict[str, Any], @@ -620,13 +638,16 @@ def _assert_transcript_matches( ``transcript`` is the fast whisper-``small`` result. If it clears ``threshold`` the check passes immediately. + If the cosine check fails, very short clips get a conservative containment + fallback that accepts minor ASR repeats/noise only when the expected text is + still present and the transcript has few extra words. + When ``escalation_model`` is set (opt-in via the ``transcript_escalation_model`` - request_config key) and the fast check fails, the clip is re-transcribed with - that stronger ASR and the assertion is decided on its verdict -- so a weak - whisper-``small`` mishear on a short clip does not flake the gate, while a - genuine model artifact still fails (the strong ASR mismatches too). When - ``escalation_model`` is ``None`` the original strict behaviour is preserved, - so other tests are unaffected. + request_config key) and the fast check plus containment fallback fail, the + clip is re-transcribed with that stronger ASR and the assertion is decided on + its verdict -- so a weak whisper-``small`` mishear on a short clip does not + flake the gate, while a genuine model artifact still fails (the strong ASR + mismatches too). """ expected = str(expected_text).strip().lower() similarity = cosine_similarity_text(transcript.strip().lower(), expected) @@ -634,6 +655,10 @@ def _assert_transcript_matches( if similarity > threshold: return + if _short_transcript_contains_expected(transcript, expected): + print("short speech containment check passed") + return + if escalation_model and audio_bytes: print( f"whisper-small below threshold ({similarity:.2f} <= {threshold}); " diff --git a/tests/helpers/test_assertions.py b/tests/helpers/test_assertions.py new file mode 100644 index 000000000..0bfa7f593 --- /dev/null +++ b/tests/helpers/test_assertions.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.helpers.assertions import _assert_transcript_matches + + +def test_short_transcript_repeat_passes_containment_fallback(): + _assert_transcript_matches( + " How... how are you?", + audio_bytes=None, + expected_text="how are you", + threshold=0.9, + ) + + +def test_short_transcript_unrelated_text_still_fails(): + with pytest.raises(AssertionError, match="Transcript doesn't match input"): + _assert_transcript_matches( + " I don't know, sorry.", + audio_bytes=None, + expected_text="how are you", + threshold=0.9, + ) diff --git a/tests/model_executor/models/voxtral_tts/test_tts_preprocess.py b/tests/model_executor/models/voxtral_tts/test_tts_preprocess.py new file mode 100644 index 000000000..399a7ecbd --- /dev/null +++ b/tests/model_executor/models/voxtral_tts/test_tts_preprocess.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import functools + +import pytest +import torch +import torch.nn as nn + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@functools.lru_cache(maxsize=1) +def _voxtral_tts_model_cls(): + from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules + + bootstrap_vllm_layer_custom_op_modules() + import vllm.model_executor.models.utils # noqa: F401 + + from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts import ( + VoxtralTTSForConditionalGeneration, + ) + + return VoxtralTTSForConditionalGeneration + + +class FakeAudioGeneration(nn.Module): + def __init__(self): + super().__init__() + self.embed_multimodal_calls = [] + + def embed_multimodal(self, **kwargs): + self.embed_multimodal_calls.append(kwargs) + return [torch.full((1, 4), 7.0)] + + +def _make_voxtral_tts_model(): + model_cls = _voxtral_tts_model_cls() + model = model_cls.__new__(model_cls) + nn.Module.__init__(model) + model.model_stage = "audio_generation" + model.model = FakeAudioGeneration() + model._audio_token_id = 42 + return model + + +def test_tts_preprocess_consumes_nested_codes_audio_feedback(): + model = _make_voxtral_tts_model() + input_ids = torch.tensor([model._audio_token_id]) + input_embeds = torch.zeros((1, 4)) + audio_tokens = torch.tensor([[1, 2, 3, 4]]) + + _, output_embeds, _ = model.tts_preprocess( + input_ids=input_ids, + input_embeds=input_embeds, + codes={"audio": audio_tokens}, + ) + + assert len(model.model.embed_multimodal_calls) == 1 + torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens) + torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0)) + + +def test_tts_preprocess_keeps_legacy_top_level_audio_feedback(): + model = _make_voxtral_tts_model() + input_ids = torch.tensor([model._audio_token_id]) + input_embeds = torch.zeros((1, 4)) + audio_tokens = torch.tensor([[5, 6, 7, 8]]) + + _, output_embeds, _ = model.tts_preprocess( + input_ids=input_ids, + input_embeds=input_embeds, + audio=audio_tokens, + ) + + assert len(model.model.embed_multimodal_calls) == 1 + torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens) + torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0)) diff --git a/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py b/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py index f262ede61..40d4ec620 100644 --- a/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py +++ b/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py @@ -209,7 +209,12 @@ class VoxtralTTSForConditionalGeneration( def tts_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict | None): self.post_process_idx = 0 - audio_tokens = info_dict.pop("audio", None) + codes = info_dict.get("codes") + audio_tokens = codes.get("audio") if isinstance(codes, Mapping) else None + if audio_tokens is None: + # Backward compatible with request state written before #4527 moved + # Voxtral TTS feedback frames under ``codes.audio``. + audio_tokens = info_dict.pop("audio", None) if audio_tokens is not None: kwargs = {"audio_tokens": audio_tokens.to(input_ids.device)} multimodal_embeddings = self.model.embed_multimodal(**kwargs)