[Bugfix] Use Seed-TTS reference audio with MiniCPM-o 4.5 (#6628)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
akshatvishu authored and GitHub committed 2026-08-31 10:27:34 +08:00
1 parent 4cca28e9bd
commit f321cdbe16
10 files changed
+529 -56

No files matched your search

+73
View File
@@ -16,9 +16,14 @@ import pytest
from pytest_mock import MockerFixture
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
SeedTTSSampleRequest,
SeedTTSTextSampleRequest,
)
from vllm_omni.benchmarks.patch.patch import (
MixRequestFuncOutput,
_apply_stage0_token_timings,
_attach_seed_tts_to_request_func_input,
async_request_openai_chat_omni_completions,
async_request_openai_realtime_duplex,
should_request_stage_metrics,
@@ -28,6 +33,74 @@ from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
def _seed_tts_request_func_input() -> RequestFuncInput:
return RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="target text",
api_url="http://test.com/v1/chat/completions",
prompt_len=2,
output_len=20,
extra_body={"modalities": ["text", "audio"]},
)
def test_seed_tts_chat_request_carries_reference_audio_once() -> None:
reference_audio_url = "data:audio/wav;base64,AAAA"
sample = SeedTTSSampleRequest(
prompt="target text",
prompt_len=2,
expected_output_len=20,
multi_modal_data=None,
seed_tts_system_prompt="Clone the supplied voice.",
seed_tts_speech_extra={
"ref_audio": reference_audio_url,
"ref_text": "reference text",
"task_type": "Base",
},
)
request_func_input = _seed_tts_request_func_input()
_attach_seed_tts_to_request_func_input(sample, request_func_input)
assert request_func_input.omni_chat_messages == [
{
"role": "system",
"content": [{"type": "text", "text": "Clone the supplied voice."}],
},
{"role": "user", "content": [{"type": "text", "text": "target text"}]},
]
assert request_func_input.extra_body["ref_audio"] == reference_audio_url
serialized_request = json.dumps(
{"messages": request_func_input.omni_chat_messages, **request_func_input.extra_body},
)
assert serialized_request.count(reference_audio_url) == 1
def test_seed_tts_text_chat_messages_do_not_add_reference_audio() -> None:
sample = SeedTTSTextSampleRequest(
prompt="target text",
prompt_len=2,
expected_output_len=20,
multi_modal_data=None,
seed_tts_system_prompt="Use the configured voice.",
seed_tts_speech_extra=None,
)
request_func_input = _seed_tts_request_func_input()
_attach_seed_tts_to_request_func_input(sample, request_func_input)
assert getattr(request_func_input, "seed_tts_row", False) is True
assert request_func_input.omni_chat_messages == [
{
"role": "system",
"content": [{"type": "text", "text": "Use the configured voice."}],
},
{"role": "user", "content": [{"type": "text", "text": "target text"}]},
]
assert "ref_audio" not in request_func_input.extra_body
class MockResponse:
"""Mock aiohttp response for testing"""
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
and SeedTTSDesignSampleRequest.
@@ -234,34 +237,6 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
assert req.seed_tts_utterance_id == "ok"
def test_attach_sets_seed_tts_row_even_without_extra_body():
"""seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
req = SeedTTSTextSampleRequest(
prompt="hello world",
prompt_len=2,
expected_output_len=100,
multi_modal_data=None,
request_id="test-0",
seed_tts_speech_extra=None,
seed_tts_ref_wav_path="",
)
assert req.seed_tts_speech_extra is None
assert req.seed_tts_ref_wav_path == ""
# The fix ensures that even with speech_extra=None, the function
# sets seed_tts_row=True. We verify the source code has the fix.
import inspect
import vllm_omni.benchmarks.patch.patch as patch_mod
src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
# seed_tts_row must be set BEFORE the 'if not ex: return' check
row_pos = src.index("seed_tts_row")
not_ex_pos = src.index("if not ex:")
assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"
def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
from vllm_omni.benchmarks.data_modules import seed_tts_eval
@@ -1,12 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output.
Exercises async chunk streaming (``--async-chunk``) across separate Thinker,
Talker, and Code2Wav stages.
"""
import json
import os
import tempfile
from copy import deepcopy
from pathlib import Path
import pytest
from vllm.logger import DEFAULT_LOGGING_CONFIG
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
@@ -23,6 +31,23 @@ _CI_DEPLOY = modify_stage_config(
},
)
_PROMPT_LOG_TEMP_DIR = tempfile.TemporaryDirectory(prefix="minicpmo45-prompt-selection-")
_PROMPT_SELECTION_LOG = Path(_PROMPT_LOG_TEMP_DIR.name) / "prompt-selection.log"
_PROMPT_LOGGING_CONFIG_PATH = Path(_PROMPT_LOG_TEMP_DIR.name) / "logging.json"
_PROMPT_LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG)
_PROMPT_LOGGING_CONFIG["handlers"]["minicpmo_prompt_selection"] = {
"class": "logging.FileHandler",
"filename": str(_PROMPT_SELECTION_LOG),
"formatter": "vllm",
"level": "DEBUG",
}
_PROMPT_LOGGING_CONFIG["loggers"]["vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav"] = {
"handlers": ["minicpmo_prompt_selection"],
"level": "DEBUG",
"propagate": True,
}
_PROMPT_LOGGING_CONFIG_PATH.write_text(json.dumps(_PROMPT_LOGGING_CONFIG), encoding="utf-8")
test_params = [
pytest.param(
OmniServerParams(
@@ -30,6 +55,10 @@ test_params = [
stage_config_path=_CI_DEPLOY,
use_stage_cli=False,
server_args=["--trust-remote-code", "--async-chunk"],
env_dict={
"VLLM_CONFIGURE_LOGGING": "1",
"VLLM_LOGGING_CONFIG_PATH": str(_PROMPT_LOGGING_CONFIG_PATH),
},
),
id="async_chunk",
),
@@ -74,6 +103,18 @@ _TTS_EXTRA_BODY = {
}
def _prompt_selection_log_offset() -> int:
return _PROMPT_SELECTION_LOG.stat().st_size if _PROMPT_SELECTION_LOG.exists() else 0
def _read_prompt_selection_log(offset: int) -> str:
if not _PROMPT_SELECTION_LOG.exists():
return ""
with open(_PROMPT_SELECTION_LOG, "rb") as log_file:
log_file.seek(offset)
return log_file.read().decode("utf-8")
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.omni
@@ -126,6 +167,51 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_audio_with_reference_audio(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
"stream": True,
"modalities": ["text", "audio"],
"extra_body": {
**_TTS_EXTRA_BODY,
"ref_audio": f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}",
},
}
log_offset = _prompt_selection_log_offset()
responses = openai_client.send_omni_request(request_config)
prompt_log = _read_prompt_selection_log(log_offset)
assert responses[0].success
assert responses[0].audio_bytes
assert "prompt_cache_id=runtime-ref-" in prompt_log
assert "minicpmo45_ref_" in prompt_log
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_audio_with_default_reference(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
"stream": True,
"modalities": ["text", "audio"],
"extra_body": _TTS_EXTRA_BODY,
}
responses = openai_client.send_omni_request(request_config)
assert responses[0].success
assert responses[0].audio_bytes
@pytest.mark.full_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@@ -0,0 +1,207 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for MiniCPM-o 4.5 chat reference audio transport."""
from __future__ import annotations
from dataclasses import dataclass, field
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import torch
from pytest_mock import MockerFixture
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.inputs import TokensPrompt
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import (
llm2tts,
tts2code2wav_async_chunk,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@dataclass
class _EngineClientStub:
stage_configs: list[dict[str, object]]
@dataclass
class _ModelConfigStub:
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
@dataclass
class _ThinkerCompletionStub:
token_ids: list[int]
text: str
hidden_states: torch.Tensor
multimodal_output: dict[str, object] = field(default_factory=dict)
@dataclass
class _ThinkerRequestOutputStub:
request_id: str
prompt_token_ids: list[int]
outputs: list[_ThinkerCompletionStub]
multimodal_output: dict[str, object] | None = None
@dataclass
class _Code2WavRequestStub:
external_req_id: str
request_id: str
model_intermediate_buffer: dict[str, object]
@dataclass
class _TransferManagerStub:
request_payload: dict[str, object] = field(default_factory=dict)
@pytest.fixture
def serving_chat() -> OmniOpenAIServingChat:
return object.__new__(OmniOpenAIServingChat)
def _stage_config(model_arch: str) -> dict[str, object]:
return {"engine_args": {"model_arch": model_arch}}
def _chat_request(reference_audio_url: str) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "target text"}],
ref_audio=reference_audio_url,
)
@pytest.mark.asyncio
async def test_chat_reference_audio_reaches_code2wav_transport(
serving_chat: OmniOpenAIServingChat,
mocker: MockerFixture,
) -> None:
reference_audio_url = "data:audio/wav;base64,AAAA"
reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
serving_chat.engine_client = _EngineClientStub(
[_stage_config("MiniCPMO45OmniForConditionalGeneration")],
)
serving_chat.model_config = _ModelConfigStub()
engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
media_connector_class = mocker.patch(
"vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
)
media_connector = media_connector_class.return_value
media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
await serving_chat._attach_minicpmo45_reference_audio(
engine_prompt,
_chat_request(reference_audio_url),
)
media_connector_class.assert_called_once_with(
media_io_kwargs=None,
allowed_local_media_path="",
allowed_media_domains=None,
)
media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY]
assert isinstance(preserved_reference, tuple)
preserved_waveform, preserved_sample_rate = preserved_reference
assert isinstance(preserved_waveform, np.ndarray)
assert isinstance(preserved_sample_rate, int)
np.testing.assert_array_equal(preserved_waveform, reference_waveform)
assert preserved_sample_rate == 16000
thinker_output = _ThinkerRequestOutputStub(
request_id="request-0",
prompt_token_ids=[10],
outputs=[
_ThinkerCompletionStub(
token_ids=[20],
text="target text",
hidden_states=torch.zeros((2, 4)),
)
],
)
talker_prompt = llm2tts([thinker_output], prompt=engine_prompt)[0]
talker_request = _Code2WavRequestStub(
external_req_id="request-0",
request_id="request-0",
model_intermediate_buffer=talker_prompt["model_intermediate_buffer"],
)
transfer_manager = _TransferManagerStub()
code2wav_payload = tts2code2wav_async_chunk(
transfer_manager,
{"codes": {"audio": torch.arange(7).reshape(-1, 1)}},
talker_request,
True,
)
assert code2wav_payload is not None
torch.testing.assert_close(code2wav_payload.codes.ref, torch.from_numpy(reference_waveform))
assert code2wav_payload.meta.ref_audio_sr == 16000
@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"])
@pytest.mark.asyncio
async def test_reference_audio_accepts_request_extra_mappings(
serving_chat: OmniOpenAIServingChat,
mocker: MockerFixture,
request_shape: str,
) -> None:
reference_audio_url = "data:audio/wav;base64,AAAA"
reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
serving_chat.engine_client = _EngineClientStub(
[_stage_config("MiniCPMO45OmniForConditionalGeneration")],
)
serving_chat.model_config = _ModelConfigStub()
engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
if request_shape == "extra_body":
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "target text"}],
extra_body={"ref_audio": reference_audio_url},
)
else:
request = SimpleNamespace(model_extra={"ref_audio": reference_audio_url})
assert not hasattr(request, "ref_audio")
media_connector = mocker.patch(
"vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
).return_value
media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
await serving_chat._attach_minicpmo45_reference_audio(engine_prompt, request)
media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY]
assert isinstance(preserved_reference, tuple)
np.testing.assert_array_equal(preserved_reference[0], reference_waveform)
assert preserved_reference[1] == 16000
@pytest.mark.asyncio
async def test_reference_audio_is_ignored_for_other_models(
serving_chat: OmniOpenAIServingChat,
mocker: MockerFixture,
) -> None:
reference_audio_url = "data:audio/wav;base64,AAAA"
serving_chat.engine_client = _EngineClientStub([_stage_config("SomeOtherOmniForConditionalGeneration")])
engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
media_connector_class = mocker.patch(
"vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
)
await serving_chat._attach_minicpmo45_reference_audio(
engine_prompt,
_chat_request(reference_audio_url),
)
media_connector_class.assert_not_called()
assert MINICPMO45_REFERENCE_AUDIO_KEY not in cast(dict[str, object], engine_prompt)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the MiniCPM-o 4.5 stage 0 -> stage 1 bridge.
Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts``:
@@ -25,6 +25,7 @@ import torch
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
MiniCPMO45OmniForConditionalGeneration,
)
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -313,6 +314,44 @@ class TestPromptAndMultiModal:
)
assert out[0]["multi_modal_data"] == mm
def test_reference_audio_is_added_to_stage_handoff(self) -> None:
hidden_states = torch.zeros((2, _HIDDEN_DIM))
waveform = torch.tensor([0.25, -0.5], dtype=torch.float32)
stage_outputs = llm2tts(
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
prompt={"multi_modal_data": {"audio": (waveform, 16000)}},
)
intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
assert intermediate_buffer["codes"]["ref"] == [0.25, -0.5]
assert intermediate_buffer["meta"]["ref_audio_sr"] == 16000
def test_serving_reference_audio_is_added_to_stage_handoff(self) -> None:
hidden_states = torch.zeros((2, _HIDDEN_DIM))
waveform = torch.tensor([0.125, -0.25], dtype=torch.float32)
stage_outputs = llm2tts(
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
prompt={MINICPMO45_REFERENCE_AUDIO_KEY: (waveform, 24000)},
)
intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
assert intermediate_buffer["codes"]["ref"] == [0.125, -0.25]
assert intermediate_buffer["meta"]["ref_audio_sr"] == 24000
def test_missing_reference_audio_does_not_add_stage_handoff_fields(self) -> None:
hidden_states = torch.zeros((2, _HIDDEN_DIM))
stage_outputs = llm2tts(
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
prompt={"multi_modal_data": {}},
)
intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
assert "codes" not in intermediate_buffer
assert "ref_audio_sr" not in intermediate_buffer.get("meta", {})
def test_internal_streaming_context_is_accepted(self) -> None:
hidden = torch.zeros((2, _HIDDEN_DIM))
out = llm2tts(
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from collections import defaultdict
from types import SimpleNamespace
@@ -168,10 +168,14 @@ def test_duplex_turn_end_waits_for_terminal_codec_flush() -> None:
def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None:
manager = _manager()
request = _request("req")
request.additional_information = {
request.model_intermediate_buffer = {
"codes": {"ref": [0.1, -0.1]},
"meta": {"ref_audio_sr": 16000},
}
request.additional_information = {
"codes": {"ref": [0.9]},
"meta": {"ref_audio_sr": 8000},
}
payload = tts2code2wav_async_chunk(
manager,
@@ -193,10 +197,33 @@ def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None:
assert payload.meta.turn_end is True
def test_first_chunk_falls_back_to_legacy_reference_fields() -> None:
manager = _manager()
request = _request("req")
request.model_intermediate_buffer = {
"meta": {"ref_audio_sr": 16000},
}
request.additional_information = {
"codes": {"ref": [0.9]},
"meta": {"ref_audio_sr": 8000},
}
payload = tts2code2wav_async_chunk(
manager,
_duplex_delta(*range(7), turn_end=True),
request,
True,
)
assert payload is not None
assert payload.codes.ref.tolist() == pytest.approx([0.9])
assert payload.meta.ref_audio_sr == 16000
def test_full_payload_forwards_all_codes_and_request_metadata() -> None:
manager = _manager()
request = _request("req")
request.additional_information = {
request.model_intermediate_buffer = {
"codes": {"ref": [0.1, -0.1]},
"meta": {
"ref_audio_sr": 16000,