[Bugfix] Use Seed-TTS reference audio with MiniCPM-o 4.5 (#6628)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
akshatvishu authored and GitHub committed 2026-08-31 10:27:34 +08:00
1 parent 4cca28e9bd
commit f321cdbe16
10 files changed
+529 -56

No files matched your search

+73
View File
@@ -16,9 +16,14 @@ import pytest
from pytest_mock import MockerFixture
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
SeedTTSSampleRequest,
SeedTTSTextSampleRequest,
)
from vllm_omni.benchmarks.patch.patch import (
MixRequestFuncOutput,
_apply_stage0_token_timings,
_attach_seed_tts_to_request_func_input,
async_request_openai_chat_omni_completions,
async_request_openai_realtime_duplex,
should_request_stage_metrics,
@@ -28,6 +33,74 @@ from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
def _seed_tts_request_func_input() -> RequestFuncInput:
return RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="target text",
api_url="http://test.com/v1/chat/completions",
prompt_len=2,
output_len=20,
extra_body={"modalities": ["text", "audio"]},
)
def test_seed_tts_chat_request_carries_reference_audio_once() -> None:
reference_audio_url = "data:audio/wav;base64,AAAA"
sample = SeedTTSSampleRequest(
prompt="target text",
prompt_len=2,
expected_output_len=20,
multi_modal_data=None,
seed_tts_system_prompt="Clone the supplied voice.",
seed_tts_speech_extra={
"ref_audio": reference_audio_url,
"ref_text": "reference text",
"task_type": "Base",
},
)
request_func_input = _seed_tts_request_func_input()
_attach_seed_tts_to_request_func_input(sample, request_func_input)
assert request_func_input.omni_chat_messages == [
{
"role": "system",
"content": [{"type": "text", "text": "Clone the supplied voice."}],
},
{"role": "user", "content": [{"type": "text", "text": "target text"}]},
]
assert request_func_input.extra_body["ref_audio"] == reference_audio_url
serialized_request = json.dumps(
{"messages": request_func_input.omni_chat_messages, **request_func_input.extra_body},
)
assert serialized_request.count(reference_audio_url) == 1
def test_seed_tts_text_chat_messages_do_not_add_reference_audio() -> None:
sample = SeedTTSTextSampleRequest(
prompt="target text",
prompt_len=2,
expected_output_len=20,
multi_modal_data=None,
seed_tts_system_prompt="Use the configured voice.",
seed_tts_speech_extra=None,
)
request_func_input = _seed_tts_request_func_input()
_attach_seed_tts_to_request_func_input(sample, request_func_input)
assert getattr(request_func_input, "seed_tts_row", False) is True
assert request_func_input.omni_chat_messages == [
{
"role": "system",
"content": [{"type": "text", "text": "Use the configured voice."}],
},
{"role": "user", "content": [{"type": "text", "text": "target text"}]},
]
assert "ref_audio" not in request_func_input.extra_body
class MockResponse:
"""Mock aiohttp response for testing"""
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
and SeedTTSDesignSampleRequest.
@@ -234,34 +237,6 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
assert req.seed_tts_utterance_id == "ok"
def test_attach_sets_seed_tts_row_even_without_extra_body():
"""seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
req = SeedTTSTextSampleRequest(
prompt="hello world",
prompt_len=2,
expected_output_len=100,
multi_modal_data=None,
request_id="test-0",
seed_tts_speech_extra=None,
seed_tts_ref_wav_path="",
)
assert req.seed_tts_speech_extra is None
assert req.seed_tts_ref_wav_path == ""
# The fix ensures that even with speech_extra=None, the function
# sets seed_tts_row=True. We verify the source code has the fix.
import inspect
import vllm_omni.benchmarks.patch.patch as patch_mod
src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
# seed_tts_row must be set BEFORE the 'if not ex: return' check
row_pos = src.index("seed_tts_row")
not_ex_pos = src.index("if not ex:")
assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
from vllm_omni.benchmarks.data_modules import seed_tts_eval