Fix Voxtral TTS feedback and short ASR checks (#4954)
Signed-off-by: leo <yaoliu548926@gmail.com>
This commit is contained in:
4 files changed
+147
-14
No files matched your search
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _voxtral_tts_model_cls():
|
||||
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
|
||||
|
||||
bootstrap_vllm_layer_custom_op_modules()
|
||||
import vllm.model_executor.models.utils # noqa: F401
|
||||
|
||||
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts import (
|
||||
VoxtralTTSForConditionalGeneration,
|
||||
)
|
||||
|
||||
return VoxtralTTSForConditionalGeneration
|
||||
|
||||
|
||||
class FakeAudioGeneration(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.embed_multimodal_calls = []
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
self.embed_multimodal_calls.append(kwargs)
|
||||
return [torch.full((1, 4), 7.0)]
|
||||
|
||||
|
||||
def _make_voxtral_tts_model():
|
||||
model_cls = _voxtral_tts_model_cls()
|
||||
model = model_cls.__new__(model_cls)
|
||||
nn.Module.__init__(model)
|
||||
model.model_stage = "audio_generation"
|
||||
model.model = FakeAudioGeneration()
|
||||
model._audio_token_id = 42
|
||||
return model
|
||||
|
||||
|
||||
def test_tts_preprocess_consumes_nested_codes_audio_feedback():
|
||||
model = _make_voxtral_tts_model()
|
||||
input_ids = torch.tensor([model._audio_token_id])
|
||||
input_embeds = torch.zeros((1, 4))
|
||||
audio_tokens = torch.tensor([[1, 2, 3, 4]])
|
||||
|
||||
_, output_embeds, _ = model.tts_preprocess(
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
codes={"audio": audio_tokens},
|
||||
)
|
||||
|
||||
assert len(model.model.embed_multimodal_calls) == 1
|
||||
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
|
||||
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
|
||||
|
||||
|
||||
def test_tts_preprocess_keeps_legacy_top_level_audio_feedback():
|
||||
model = _make_voxtral_tts_model()
|
||||
input_ids = torch.tensor([model._audio_token_id])
|
||||
input_embeds = torch.zeros((1, 4))
|
||||
audio_tokens = torch.tensor([[5, 6, 7, 8]])
|
||||
|
||||
_, output_embeds, _ = model.tts_preprocess(
|
||||
input_ids=input_ids,
|
||||
input_embeds=input_embeds,
|
||||
audio=audio_tokens,
|
||||
)
|
||||
|
||||
assert len(model.model.embed_multimodal_calls) == 1
|
||||
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
|
||||
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
|
||||
Reference in new issue
Block a user