Fix Voxtral TTS feedback and short ASR checks (#4954)
Signed-off-by: leo <yaoliu548926@gmail.com>
This commit is contained in:
4 files changed
+147
-14
No files matched your search
+38
-13
@@ -19,19 +19,20 @@ from PIL import Image
|
||||
from tests.helpers.media import (
|
||||
convert_audio_bytes_to_text,
|
||||
cosine_similarity_text,
|
||||
preprocess_text,
|
||||
)
|
||||
|
||||
_GENDER_PIPELINE = None
|
||||
_GENDER_PIPELINE_LOCK = threading.Lock()
|
||||
# Transcript gates default to whisper ``small`` for speed. ``small`` mishears a
|
||||
# short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a
|
||||
# leading SFX token), which flakes the deterministic similarity gate. A test can
|
||||
# opt in to ASR escalation by setting ``transcript_escalation_model`` to a
|
||||
# whisper model name (e.g. ``"large-v3"``) in its request_config: on a failed
|
||||
# fast pass the clip is re-transcribed with that stronger ASR before the test
|
||||
# fails, so a weak-ASR mishear is rescued while a genuine model artifact still
|
||||
# fails (the strong ASR mismatches too). Tests that do not opt in keep the
|
||||
# strict behaviour.
|
||||
# leading SFX token), which flakes the deterministic similarity gate. Short
|
||||
# clips first get a conservative containment fallback for minor ASR repeats/noise.
|
||||
# A test can also opt in to ASR escalation by setting
|
||||
# ``transcript_escalation_model`` to a whisper model name (e.g. ``"large-v3"``)
|
||||
# in its request_config: on a failed fast pass the clip is re-transcribed with
|
||||
# that stronger ASR before the test fails, so a weak-ASR mishear is rescued while
|
||||
# a genuine model artifact still fails (the strong ASR mismatches too).
|
||||
_PCM_SPEECH_SAMPLE_RATE_HZ = 24_000
|
||||
_MIN_PCM_SPEECH_HNR_DB = 1.0
|
||||
_PRESET_VOICE_GENDER_MAP: dict[str, str] = {
|
||||
@@ -43,6 +44,23 @@ _PRESET_VOICE_GENDER_MAP: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
def _short_transcript_contains_expected(transcript: str, expected: str) -> bool:
|
||||
"""Allow minor ASR repeats/noise for very short speech clips."""
|
||||
transcript_clean = preprocess_text(transcript)
|
||||
expected_clean = preprocess_text(expected)
|
||||
if not transcript_clean or not expected_clean:
|
||||
return False
|
||||
|
||||
transcript_words = transcript_clean.split()
|
||||
expected_words = expected_clean.split()
|
||||
if not transcript_words or not expected_words:
|
||||
return False
|
||||
|
||||
short_text = min(len(transcript_clean), len(expected_clean)) <= 15
|
||||
small_word_delta = len(transcript_words) <= len(expected_words) + 2
|
||||
return short_text and small_word_delta and expected_clean in transcript_clean
|
||||
|
||||
|
||||
def assert_image_diffusion_response(
|
||||
response: "DiffusionResponse",
|
||||
request_config: dict[str, Any],
|
||||
@@ -620,13 +638,16 @@ def _assert_transcript_matches(
|
||||
``transcript`` is the fast whisper-``small`` result. If it clears
|
||||
``threshold`` the check passes immediately.
|
||||
|
||||
If the cosine check fails, very short clips get a conservative containment
|
||||
fallback that accepts minor ASR repeats/noise only when the expected text is
|
||||
still present and the transcript has few extra words.
|
||||
|
||||
When ``escalation_model`` is set (opt-in via the ``transcript_escalation_model``
|
||||
request_config key) and the fast check fails, the clip is re-transcribed with
|
||||
that stronger ASR and the assertion is decided on its verdict -- so a weak
|
||||
whisper-``small`` mishear on a short clip does not flake the gate, while a
|
||||
genuine model artifact still fails (the strong ASR mismatches too). When
|
||||
``escalation_model`` is ``None`` the original strict behaviour is preserved,
|
||||
so other tests are unaffected.
|
||||
request_config key) and the fast check plus containment fallback fail, the
|
||||
clip is re-transcribed with that stronger ASR and the assertion is decided on
|
||||
its verdict -- so a weak whisper-``small`` mishear on a short clip does not
|
||||
flake the gate, while a genuine model artifact still fails (the strong ASR
|
||||
mismatches too).
|
||||
"""
|
||||
expected = str(expected_text).strip().lower()
|
||||
similarity = cosine_similarity_text(transcript.strip().lower(), expected)
|
||||
@@ -634,6 +655,10 @@ def _assert_transcript_matches(
|
||||
if similarity > threshold:
|
||||
return
|
||||
|
||||
if _short_transcript_contains_expected(transcript, expected):
|
||||
print("short speech containment check passed")
|
||||
return
|
||||
|
||||
if escalation_model and audio_bytes:
|
||||
print(
|
||||
f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.assertions import _assert_transcript_matches
|
||||
|
||||
|
||||
def test_short_transcript_repeat_passes_containment_fallback():
|
||||
_assert_transcript_matches(
|
||||
" How... how are you?",
|
||||
audio_bytes=None,
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
)
|
||||
|
||||
|
||||
def test_short_transcript_unrelated_text_still_fails():
|
||||
with pytest.raises(AssertionError, match="Transcript doesn't match input"):
|
||||
_assert_transcript_matches(
|
||||
" I don't know, sorry.",
|
||||
audio_bytes=None,
|
||||
expected_text="how are you",
|
||||
threshold=0.9,
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _voxtral_tts_model_cls():
|
||||
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
|
||||
|
||||
bootstrap_vllm_layer_custom_op_modules()
|
||||
import vllm.model_executor.models.utils # noqa: F401
|
||||
|
||||
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts import (
|
||||
VoxtralTTSForConditionalGeneration,
|
||||
)
|
||||
|
||||
return VoxtralTTSForConditionalGeneration
|
||||
|
||||
|
||||
class FakeAudioGeneration(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.embed_multimodal_calls = []
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
self.embed_multimodal_calls.append(kwargs)
|
||||
return [torch.full((1, 4), 7.0)]
|
||||
|
||||
|
||||
def _make_voxtral_tts_model():
|
||||
model_cls = _voxtral_tts_model_cls()
|
||||
model = model_cls.__new__(model_cls)
|
||||
nn.Module.__init__(model)
|
||||
model.model_stage = "audio_generation"
|
||||
model.model = FakeAudioGeneration()
|
||||
model._audio_token_id = 42
|
||||
return model
|
||||
|
||||
|
||||
def test_tts_preprocess_consumes_nested_codes_audio_feedback():
|
||||
model = _make_voxtral_tts_model()
|
||||
input_ids = torch.tensor([model._audio_token_id])
|
||||
input_embeds = torch.zeros((1, 4))
|
||||
audio_tokens = torch.tensor([[1, 2, 3, 4]])
|
||||
|
||||
_, output_embeds, _ = model.tts_preprocess(
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
codes={"audio": audio_tokens},
|
||||
)
|
||||
|
||||
assert len(model.model.embed_multimodal_calls) == 1
|
||||
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
|
||||
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
|
||||
|
||||
|
||||
def test_tts_preprocess_keeps_legacy_top_level_audio_feedback():
|
||||
model = _make_voxtral_tts_model()
|
||||
input_ids = torch.tensor([model._audio_token_id])
|
||||
input_embeds = torch.zeros((1, 4))
|
||||
audio_tokens = torch.tensor([[5, 6, 7, 8]])
|
||||
|
||||
_, output_embeds, _ = model.tts_preprocess(
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
audio=audio_tokens,
|
||||
)
|
||||
|
||||
assert len(model.model.embed_multimodal_calls) == 1
|
||||
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
|
||||
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
|
||||
@@ -209,7 +209,12 @@ class VoxtralTTSForConditionalGeneration(
|
||||
|
||||
def tts_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict | None):
|
||||
self.post_process_idx = 0
|
||||
audio_tokens = info_dict.pop("audio", None)
|
||||
codes = info_dict.get("codes")
|
||||
audio_tokens = codes.get("audio") if isinstance(codes, Mapping) else None
|
||||
if audio_tokens is None:
|
||||
# Backward compatible with request state written before #4527 moved
|
||||
# Voxtral TTS feedback frames under ``codes.audio``.
|
||||
audio_tokens = info_dict.pop("audio", None)
|
||||
if audio_tokens is not None:
|
||||
kwargs = {"audio_tokens": audio_tokens.to(input_ids.device)}
|
||||
multimodal_embeddings = self.model.embed_multimodal(**kwargs)
|
||||
|
||||
Reference in new issue
Block a user