Fix Voxtral TTS feedback and short ASR checks (#4954)

Signed-off-by: leo <yaoliu548926@gmail.com>
This commit is contained in:
leo authored and GitHub committed 2026-07-08 16:52:48 +00:00
1 parent b7d08fe5f6
commit 267b49b880
4 files changed
+147 -14

No files matched your search

+38 -13
View File
@@ -19,19 +19,20 @@ from PIL import Image
from tests.helpers.media import (
convert_audio_bytes_to_text,
cosine_similarity_text,
preprocess_text,
)
_GENDER_PIPELINE = None
_GENDER_PIPELINE_LOCK = threading.Lock()
# Transcript gates default to whisper ``small`` for speed. ``small`` mishears a
# short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a
# leading SFX token), which flakes the deterministic similarity gate. A test can
# opt in to ASR escalation by setting ``transcript_escalation_model`` to a
# whisper model name (e.g. ``"large-v3"``) in its request_config: on a failed
# fast pass the clip is re-transcribed with that stronger ASR before the test
# fails, so a weak-ASR mishear is rescued while a genuine model artifact still
# fails (the strong ASR mismatches too). Tests that do not opt in keep the
# strict behaviour.
# leading SFX token), which flakes the deterministic similarity gate. Short
# clips first get a conservative containment fallback for minor ASR repeats/noise.
# A test can also opt in to ASR escalation by setting
# ``transcript_escalation_model`` to a whisper model name (e.g. ``"large-v3"``)
# in its request_config: on a failed fast pass the clip is re-transcribed with
# that stronger ASR before the test fails, so a weak-ASR mishear is rescued while
# a genuine model artifact still fails (the strong ASR mismatches too).
_PCM_SPEECH_SAMPLE_RATE_HZ = 24_000
_MIN_PCM_SPEECH_HNR_DB = 1.0
_PRESET_VOICE_GENDER_MAP: dict[str, str] = {
@@ -43,6 +44,23 @@ _PRESET_VOICE_GENDER_MAP: dict[str, str] = {
}
def _short_transcript_contains_expected(transcript: str, expected: str) -> bool:
"""Allow minor ASR repeats/noise for very short speech clips."""
transcript_clean = preprocess_text(transcript)
expected_clean = preprocess_text(expected)
if not transcript_clean or not expected_clean:
return False
transcript_words = transcript_clean.split()
expected_words = expected_clean.split()
if not transcript_words or not expected_words:
return False
short_text = min(len(transcript_clean), len(expected_clean)) <= 15
small_word_delta = len(transcript_words) <= len(expected_words) + 2
return short_text and small_word_delta and expected_clean in transcript_clean
def assert_image_diffusion_response(
response: "DiffusionResponse",
request_config: dict[str, Any],
@@ -620,13 +638,16 @@ def _assert_transcript_matches(
``transcript`` is the fast whisper-``small`` result. If it clears
``threshold`` the check passes immediately.
If the cosine check fails, very short clips get a conservative containment
fallback that accepts minor ASR repeats/noise only when the expected text is
still present and the transcript has few extra words.
When ``escalation_model`` is set (opt-in via the ``transcript_escalation_model``
request_config key) and the fast check fails, the clip is re-transcribed with
that stronger ASR and the assertion is decided on its verdict -- so a weak
whisper-``small`` mishear on a short clip does not flake the gate, while a
genuine model artifact still fails (the strong ASR mismatches too). When
``escalation_model`` is ``None`` the original strict behaviour is preserved,
so other tests are unaffected.
request_config key) and the fast check plus containment fallback fail, the
clip is re-transcribed with that stronger ASR and the assertion is decided on
its verdict -- so a weak whisper-``small`` mishear on a short clip does not
flake the gate, while a genuine model artifact still fails (the strong ASR
mismatches too).
"""
expected = str(expected_text).strip().lower()
similarity = cosine_similarity_text(transcript.strip().lower(), expected)
@@ -634,6 +655,10 @@ def _assert_transcript_matches(
if similarity > threshold:
return
if _short_transcript_contains_expected(transcript, expected):
print("short speech containment check passed")
return
if escalation_model and audio_bytes:
print(
f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
+25
View File
@@ -0,0 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.helpers.assertions import _assert_transcript_matches
def test_short_transcript_repeat_passes_containment_fallback():
_assert_transcript_matches(
" How... how are you?",
audio_bytes=None,
expected_text="how are you",
threshold=0.9,
)
def test_short_transcript_unrelated_text_still_fails():
with pytest.raises(AssertionError, match="Transcript doesn't match input"):
_assert_transcript_matches(
" I don't know, sorry.",
audio_bytes=None,
expected_text="how are you",
threshold=0.9,
)
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import pytest
import torch
import torch.nn as nn
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@functools.lru_cache(maxsize=1)
def _voxtral_tts_model_cls():
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
bootstrap_vllm_layer_custom_op_modules()
import vllm.model_executor.models.utils # noqa: F401
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts import (
VoxtralTTSForConditionalGeneration,
)
return VoxtralTTSForConditionalGeneration
class FakeAudioGeneration(nn.Module):
def __init__(self):
super().__init__()
self.embed_multimodal_calls = []
def embed_multimodal(self, **kwargs):
self.embed_multimodal_calls.append(kwargs)
return [torch.full((1, 4), 7.0)]
def _make_voxtral_tts_model():
model_cls = _voxtral_tts_model_cls()
model = model_cls.__new__(model_cls)
nn.Module.__init__(model)
model.model_stage = "audio_generation"
model.model = FakeAudioGeneration()
model._audio_token_id = 42
return model
def test_tts_preprocess_consumes_nested_codes_audio_feedback():
model = _make_voxtral_tts_model()
input_ids = torch.tensor([model._audio_token_id])
input_embeds = torch.zeros((1, 4))
audio_tokens = torch.tensor([[1, 2, 3, 4]])
_, output_embeds, _ = model.tts_preprocess(
input_ids=input_ids,
input_embeds=input_embeds,
codes={"audio": audio_tokens},
)
assert len(model.model.embed_multimodal_calls) == 1
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
def test_tts_preprocess_keeps_legacy_top_level_audio_feedback():
model = _make_voxtral_tts_model()
input_ids = torch.tensor([model._audio_token_id])
input_embeds = torch.zeros((1, 4))
audio_tokens = torch.tensor([[5, 6, 7, 8]])
_, output_embeds, _ = model.tts_preprocess(
input_ids=input_ids,
input_embeds=input_embeds,
audio=audio_tokens,
)
assert len(model.model.embed_multimodal_calls) == 1
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
@@ -209,7 +209,12 @@ class VoxtralTTSForConditionalGeneration(
def tts_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict | None):
self.post_process_idx = 0
audio_tokens = info_dict.pop("audio", None)
codes = info_dict.get("codes")
audio_tokens = codes.get("audio") if isinstance(codes, Mapping) else None
if audio_tokens is None:
# Backward compatible with request state written before #4527 moved
# Voxtral TTS feedback frames under ``codes.audio``.
audio_tokens = info_dict.pop("audio", None)
if audio_tokens is not None:
kwargs = {"audio_tokens": audio_tokens.to(input_ids.device)}
multimodal_embeddings = self.model.embed_multimodal(**kwargs)