Fix Voxtral TTS feedback and short ASR checks (#4954)

Signed-off-by: leo <yaoliu548926@gmail.com>
This commit is contained in:
leo authored and GitHub committed 2026-07-08 16:52:48 +00:00
1 parent b7d08fe5f6
commit 267b49b880
4 files changed
+147 -14

No files matched your search

@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import pytest
import torch
import torch.nn as nn
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@functools.lru_cache(maxsize=1)
def _voxtral_tts_model_cls():
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
bootstrap_vllm_layer_custom_op_modules()
import vllm.model_executor.models.utils # noqa: F401
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts import (
VoxtralTTSForConditionalGeneration,
)
return VoxtralTTSForConditionalGeneration
class FakeAudioGeneration(nn.Module):
def __init__(self):
super().__init__()
self.embed_multimodal_calls = []
def embed_multimodal(self, **kwargs):
self.embed_multimodal_calls.append(kwargs)
return [torch.full((1, 4), 7.0)]
def _make_voxtral_tts_model():
model_cls = _voxtral_tts_model_cls()
model = model_cls.__new__(model_cls)
nn.Module.__init__(model)
model.model_stage = "audio_generation"
model.model = FakeAudioGeneration()
model._audio_token_id = 42
return model
def test_tts_preprocess_consumes_nested_codes_audio_feedback():
model = _make_voxtral_tts_model()
input_ids = torch.tensor([model._audio_token_id])
input_embeds = torch.zeros((1, 4))
audio_tokens = torch.tensor([[1, 2, 3, 4]])
_, output_embeds, _ = model.tts_preprocess(
input_ids=input_ids,
input_embeds=input_embeds,
codes={"audio": audio_tokens},
)
assert len(model.model.embed_multimodal_calls) == 1
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))
def test_tts_preprocess_keeps_legacy_top_level_audio_feedback():
model = _make_voxtral_tts_model()
input_ids = torch.tensor([model._audio_token_id])
input_embeds = torch.zeros((1, 4))
audio_tokens = torch.tensor([[5, 6, 7, 8]])
_, output_embeds, _ = model.tts_preprocess(
input_ids=input_ids,
input_embeds=input_embeds,
audio=audio_tokens,
)
assert len(model.model.embed_multimodal_calls) == 1
torch.testing.assert_close(model.model.embed_multimodal_calls[0]["audio_tokens"], audio_tokens)
torch.testing.assert_close(output_embeds, torch.full((1, 4), 7.0))