From f321cdbe16df69da4f1770decc500ad55ed764ac Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Mon, 31 Aug 2026 07:57:34 +0530 Subject: [PATCH] [Bugfix] Use Seed-TTS reference audio with MiniCPM-o 4.5 (#6628) Signed-off-by: akshatvishu --- tests/benchmarks/patch/test_patch.py | 73 ++++++ .../test_seed_tts_dataset_variants.py | 31 +-- tests/e2e/online_serving/test_minicpmo_4_5.py | 86 ++++++++ ...serving_chat_minicpmo45_reference_audio.py | 207 ++++++++++++++++++ .../models/minicpmo_4_5/test_llm2tts.py | 41 +++- .../test_minicpmo_4_5_async_chunk.py | 33 ++- vllm_omni/entrypoints/openai/serving_chat.py | 41 ++++ .../minicpmo_4_5/minicpmo_4_5_code2wav.py | 5 + .../models/minicpmo_4_5/pipeline.py | 3 +- .../minicpmo_4_5_omni.py | 65 ++++-- 10 files changed, 529 insertions(+), 56 deletions(-) create mode 100644 tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py diff --git a/tests/benchmarks/patch/test_patch.py b/tests/benchmarks/patch/test_patch.py index ad33a03ab..d5c714be8 100644 --- a/tests/benchmarks/patch/test_patch.py +++ b/tests/benchmarks/patch/test_patch.py @@ -16,9 +16,14 @@ import pytest from pytest_mock import MockerFixture from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput +from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( + SeedTTSSampleRequest, + SeedTTSTextSampleRequest, +) from vllm_omni.benchmarks.patch.patch import ( MixRequestFuncOutput, _apply_stage0_token_timings, + _attach_seed_tts_to_request_func_input, async_request_openai_chat_omni_completions, async_request_openai_realtime_duplex, should_request_stage_metrics, @@ -28,6 +33,74 @@ from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu] +def _seed_tts_request_func_input() -> RequestFuncInput: + return RequestFuncInput( + model="test-model", + model_name="test-model", + prompt="target text", + api_url="http://test.com/v1/chat/completions", + prompt_len=2, + output_len=20, + extra_body={"modalities": ["text", "audio"]}, + ) + + +def test_seed_tts_chat_request_carries_reference_audio_once() -> None: + reference_audio_url = "data:audio/wav;base64,AAAA" + sample = SeedTTSSampleRequest( + prompt="target text", + prompt_len=2, + expected_output_len=20, + multi_modal_data=None, + seed_tts_system_prompt="Clone the supplied voice.", + seed_tts_speech_extra={ + "ref_audio": reference_audio_url, + "ref_text": "reference text", + "task_type": "Base", + }, + ) + request_func_input = _seed_tts_request_func_input() + + _attach_seed_tts_to_request_func_input(sample, request_func_input) + + assert request_func_input.omni_chat_messages == [ + { + "role": "system", + "content": [{"type": "text", "text": "Clone the supplied voice."}], + }, + {"role": "user", "content": [{"type": "text", "text": "target text"}]}, + ] + assert request_func_input.extra_body["ref_audio"] == reference_audio_url + serialized_request = json.dumps( + {"messages": request_func_input.omni_chat_messages, **request_func_input.extra_body}, + ) + assert serialized_request.count(reference_audio_url) == 1 + + +def test_seed_tts_text_chat_messages_do_not_add_reference_audio() -> None: + sample = SeedTTSTextSampleRequest( + prompt="target text", + prompt_len=2, + expected_output_len=20, + multi_modal_data=None, + seed_tts_system_prompt="Use the configured voice.", + seed_tts_speech_extra=None, + ) + request_func_input = _seed_tts_request_func_input() + + _attach_seed_tts_to_request_func_input(sample, request_func_input) + + assert getattr(request_func_input, "seed_tts_row", False) is True + assert request_func_input.omni_chat_messages == [ + { + "role": "system", + "content": [{"type": "text", "text": "Use the configured voice."}], + }, + {"role": "user", "content": [{"type": "text", "text": "target text"}]}, + ] + assert "ref_audio" not in request_func_input.extra_body + + class MockResponse: """Mock aiohttp response for testing""" diff --git a/tests/benchmarks/test_seed_tts_dataset_variants.py b/tests/benchmarks/test_seed_tts_dataset_variants.py index fe1e6b62f..31f66bd7d 100644 --- a/tests/benchmarks/test_seed_tts_dataset_variants.py +++ b/tests/benchmarks/test_seed_tts_dataset_variants.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset, and SeedTTSDesignSampleRequest. @@ -234,34 +237,6 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo assert req.seed_tts_utterance_id == "ok" -def test_attach_sets_seed_tts_row_even_without_extra_body(): - """seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body).""" - from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest - - req = SeedTTSTextSampleRequest( - prompt="hello world", - prompt_len=2, - expected_output_len=100, - multi_modal_data=None, - request_id="test-0", - seed_tts_speech_extra=None, - seed_tts_ref_wav_path="", - ) - assert req.seed_tts_speech_extra is None - assert req.seed_tts_ref_wav_path == "" - # The fix ensures that even with speech_extra=None, the function - # sets seed_tts_row=True. We verify the source code has the fix. - import inspect - - import vllm_omni.benchmarks.patch.patch as patch_mod - - src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input) - # seed_tts_row must be set BEFORE the 'if not ex: return' check - row_pos = src.index("seed_tts_row") - not_ex_pos = src.index("if not ex:") - assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'" - - def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch): from vllm_omni.benchmarks.data_modules import seed_tts_eval diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index 9c8bcd908..4ba411738 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -1,12 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output. Exercises async chunk streaming (``--async-chunk``) across separate Thinker, Talker, and Code2Wav stages. """ +import json import os +import tempfile +from copy import deepcopy +from pathlib import Path import pytest +from vllm.logger import DEFAULT_LOGGING_CONFIG from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video @@ -23,6 +31,23 @@ _CI_DEPLOY = modify_stage_config( }, ) +_PROMPT_LOG_TEMP_DIR = tempfile.TemporaryDirectory(prefix="minicpmo45-prompt-selection-") +_PROMPT_SELECTION_LOG = Path(_PROMPT_LOG_TEMP_DIR.name) / "prompt-selection.log" +_PROMPT_LOGGING_CONFIG_PATH = Path(_PROMPT_LOG_TEMP_DIR.name) / "logging.json" +_PROMPT_LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG) +_PROMPT_LOGGING_CONFIG["handlers"]["minicpmo_prompt_selection"] = { + "class": "logging.FileHandler", + "filename": str(_PROMPT_SELECTION_LOG), + "formatter": "vllm", + "level": "DEBUG", +} +_PROMPT_LOGGING_CONFIG["loggers"]["vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav"] = { + "handlers": ["minicpmo_prompt_selection"], + "level": "DEBUG", + "propagate": True, +} +_PROMPT_LOGGING_CONFIG_PATH.write_text(json.dumps(_PROMPT_LOGGING_CONFIG), encoding="utf-8") + test_params = [ pytest.param( OmniServerParams( @@ -30,6 +55,10 @@ test_params = [ stage_config_path=_CI_DEPLOY, use_stage_cli=False, server_args=["--trust-remote-code", "--async-chunk"], + env_dict={ + "VLLM_CONFIGURE_LOGGING": "1", + "VLLM_LOGGING_CONFIG_PATH": str(_PROMPT_LOGGING_CONFIG_PATH), + }, ), id="async_chunk", ), @@ -74,6 +103,18 @@ _TTS_EXTRA_BODY = { } +def _prompt_selection_log_offset() -> int: + return _PROMPT_SELECTION_LOG.stat().st_size if _PROMPT_SELECTION_LOG.exists() else 0 + + +def _read_prompt_selection_log(offset: int) -> str: + if not _PROMPT_SELECTION_LOG.exists(): + return "" + with open(_PROMPT_SELECTION_LOG, "rb") as log_file: + log_file.seek(offset) + return log_file.read().decode("utf-8") + + @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni @@ -126,6 +167,51 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_with_reference_audio(omni_server, openai_client) -> None: + request_config = { + "model": omni_server.model, + "messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()), + "stream": True, + "modalities": ["text", "audio"], + "extra_body": { + **_TTS_EXTRA_BODY, + "ref_audio": f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}", + }, + } + + log_offset = _prompt_selection_log_offset() + responses = openai_client.send_omni_request(request_config) + prompt_log = _read_prompt_selection_log(log_offset) + + assert responses[0].success + assert responses[0].audio_bytes + assert "prompt_cache_id=runtime-ref-" in prompt_log + assert "minicpmo45_ref_" in prompt_log + + +@pytest.mark.advanced_model +@pytest.mark.omni +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_audio_with_default_reference(omni_server, openai_client) -> None: + request_config = { + "model": omni_server.model, + "messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()), + "stream": True, + "modalities": ["text", "audio"], + "extra_body": _TTS_EXTRA_BODY, + } + + responses = openai_client.send_omni_request(request_config) + + assert responses[0].success + assert responses[0].audio_bytes + + @pytest.mark.full_model @pytest.mark.omni @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) diff --git a/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py b/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py new file mode 100644 index 000000000..59d070a39 --- /dev/null +++ b/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Regression tests for MiniCPM-o 4.5 chat reference audio transport.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import cast + +import numpy as np +import pytest +import torch +from pytest_mock import MockerFixture +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.inputs import TokensPrompt + +from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat +from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY +from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import ( + llm2tts, + tts2code2wav_async_chunk, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@dataclass +class _EngineClientStub: + stage_configs: list[dict[str, object]] + + +@dataclass +class _ModelConfigStub: + allowed_local_media_path: str = "" + allowed_media_domains: list[str] | None = None + + +@dataclass +class _ThinkerCompletionStub: + token_ids: list[int] + text: str + hidden_states: torch.Tensor + multimodal_output: dict[str, object] = field(default_factory=dict) + + +@dataclass +class _ThinkerRequestOutputStub: + request_id: str + prompt_token_ids: list[int] + outputs: list[_ThinkerCompletionStub] + multimodal_output: dict[str, object] | None = None + + +@dataclass +class _Code2WavRequestStub: + external_req_id: str + request_id: str + model_intermediate_buffer: dict[str, object] + + +@dataclass +class _TransferManagerStub: + request_payload: dict[str, object] = field(default_factory=dict) + + +@pytest.fixture +def serving_chat() -> OmniOpenAIServingChat: + return object.__new__(OmniOpenAIServingChat) + + +def _stage_config(model_arch: str) -> dict[str, object]: + return {"engine_args": {"model_arch": model_arch}} + + +def _chat_request(reference_audio_url: str) -> ChatCompletionRequest: + return ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "target text"}], + ref_audio=reference_audio_url, + ) + + +@pytest.mark.asyncio +async def test_chat_reference_audio_reaches_code2wav_transport( + serving_chat: OmniOpenAIServingChat, + mocker: MockerFixture, +) -> None: + reference_audio_url = "data:audio/wav;base64,AAAA" + reference_waveform = np.array([0.25, -0.5], dtype=np.float32) + serving_chat.engine_client = _EngineClientStub( + [_stage_config("MiniCPMO45OmniForConditionalGeneration")], + ) + serving_chat.model_config = _ModelConfigStub() + engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]} + media_connector_class = mocker.patch( + "vllm_omni.entrypoints.openai.serving_chat.MediaConnector", + ) + media_connector = media_connector_class.return_value + media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000)) + + await serving_chat._attach_minicpmo45_reference_audio( + engine_prompt, + _chat_request(reference_audio_url), + ) + + media_connector_class.assert_called_once_with( + media_io_kwargs=None, + allowed_local_media_path="", + allowed_media_domains=None, + ) + media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url) + preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY] + assert isinstance(preserved_reference, tuple) + preserved_waveform, preserved_sample_rate = preserved_reference + assert isinstance(preserved_waveform, np.ndarray) + assert isinstance(preserved_sample_rate, int) + np.testing.assert_array_equal(preserved_waveform, reference_waveform) + assert preserved_sample_rate == 16000 + + thinker_output = _ThinkerRequestOutputStub( + request_id="request-0", + prompt_token_ids=[10], + outputs=[ + _ThinkerCompletionStub( + token_ids=[20], + text="target text", + hidden_states=torch.zeros((2, 4)), + ) + ], + ) + talker_prompt = llm2tts([thinker_output], prompt=engine_prompt)[0] + talker_request = _Code2WavRequestStub( + external_req_id="request-0", + request_id="request-0", + model_intermediate_buffer=talker_prompt["model_intermediate_buffer"], + ) + transfer_manager = _TransferManagerStub() + + code2wav_payload = tts2code2wav_async_chunk( + transfer_manager, + {"codes": {"audio": torch.arange(7).reshape(-1, 1)}}, + talker_request, + True, + ) + + assert code2wav_payload is not None + torch.testing.assert_close(code2wav_payload.codes.ref, torch.from_numpy(reference_waveform)) + assert code2wav_payload.meta.ref_audio_sr == 16000 + + +@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"]) +@pytest.mark.asyncio +async def test_reference_audio_accepts_request_extra_mappings( + serving_chat: OmniOpenAIServingChat, + mocker: MockerFixture, + request_shape: str, +) -> None: + reference_audio_url = "data:audio/wav;base64,AAAA" + reference_waveform = np.array([0.25, -0.5], dtype=np.float32) + serving_chat.engine_client = _EngineClientStub( + [_stage_config("MiniCPMO45OmniForConditionalGeneration")], + ) + serving_chat.model_config = _ModelConfigStub() + engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]} + if request_shape == "extra_body": + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "target text"}], + extra_body={"ref_audio": reference_audio_url}, + ) + else: + request = SimpleNamespace(model_extra={"ref_audio": reference_audio_url}) + assert not hasattr(request, "ref_audio") + media_connector = mocker.patch( + "vllm_omni.entrypoints.openai.serving_chat.MediaConnector", + ).return_value + media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000)) + + await serving_chat._attach_minicpmo45_reference_audio(engine_prompt, request) + + media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url) + preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY] + assert isinstance(preserved_reference, tuple) + np.testing.assert_array_equal(preserved_reference[0], reference_waveform) + assert preserved_reference[1] == 16000 + + +@pytest.mark.asyncio +async def test_reference_audio_is_ignored_for_other_models( + serving_chat: OmniOpenAIServingChat, + mocker: MockerFixture, +) -> None: + reference_audio_url = "data:audio/wav;base64,AAAA" + serving_chat.engine_client = _EngineClientStub([_stage_config("SomeOtherOmniForConditionalGeneration")]) + engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]} + media_connector_class = mocker.patch( + "vllm_omni.entrypoints.openai.serving_chat.MediaConnector", + ) + + await serving_chat._attach_minicpmo45_reference_audio( + engine_prompt, + _chat_request(reference_audio_url), + ) + + media_connector_class.assert_not_called() + assert MINICPMO45_REFERENCE_AUDIO_KEY not in cast(dict[str, object], engine_prompt) diff --git a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py index 9c4b50967..3fd4867cb 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py +++ b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Unit tests for the MiniCPM-o 4.5 stage 0 -> stage 1 bridge. Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts``: @@ -25,6 +25,7 @@ import torch from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( MiniCPMO45OmniForConditionalGeneration, ) +from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -313,6 +314,44 @@ class TestPromptAndMultiModal: ) assert out[0]["multi_modal_data"] == mm + def test_reference_audio_is_added_to_stage_handoff(self) -> None: + hidden_states = torch.zeros((2, _HIDDEN_DIM)) + waveform = torch.tensor([0.25, -0.5], dtype=torch.float32) + + stage_outputs = llm2tts( + [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)], + prompt={"multi_modal_data": {"audio": (waveform, 16000)}}, + ) + + intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"] + assert intermediate_buffer["codes"]["ref"] == [0.25, -0.5] + assert intermediate_buffer["meta"]["ref_audio_sr"] == 16000 + + def test_serving_reference_audio_is_added_to_stage_handoff(self) -> None: + hidden_states = torch.zeros((2, _HIDDEN_DIM)) + waveform = torch.tensor([0.125, -0.25], dtype=torch.float32) + + stage_outputs = llm2tts( + [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)], + prompt={MINICPMO45_REFERENCE_AUDIO_KEY: (waveform, 24000)}, + ) + + intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"] + assert intermediate_buffer["codes"]["ref"] == [0.125, -0.25] + assert intermediate_buffer["meta"]["ref_audio_sr"] == 24000 + + def test_missing_reference_audio_does_not_add_stage_handoff_fields(self) -> None: + hidden_states = torch.zeros((2, _HIDDEN_DIM)) + + stage_outputs = llm2tts( + [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)], + prompt={"multi_modal_data": {}}, + ) + + intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"] + assert "codes" not in intermediate_buffer + assert "ref_audio_sr" not in intermediate_buffer.get("meta", {}) + def test_internal_streaming_context_is_accepted(self) -> None: hidden = torch.zeros((2, _HIDDEN_DIM)) out = llm2tts( diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py index fbc3dd409..afa7b46c4 100644 --- a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from collections import defaultdict from types import SimpleNamespace @@ -168,10 +168,14 @@ def test_duplex_turn_end_waits_for_terminal_codec_flush() -> None: def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None: manager = _manager() request = _request("req") - request.additional_information = { + request.model_intermediate_buffer = { "codes": {"ref": [0.1, -0.1]}, "meta": {"ref_audio_sr": 16000}, } + request.additional_information = { + "codes": {"ref": [0.9]}, + "meta": {"ref_audio_sr": 8000}, + } payload = tts2code2wav_async_chunk( manager, @@ -193,10 +197,33 @@ def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None: assert payload.meta.turn_end is True +def test_first_chunk_falls_back_to_legacy_reference_fields() -> None: + manager = _manager() + request = _request("req") + request.model_intermediate_buffer = { + "meta": {"ref_audio_sr": 16000}, + } + request.additional_information = { + "codes": {"ref": [0.9]}, + "meta": {"ref_audio_sr": 8000}, + } + + payload = tts2code2wav_async_chunk( + manager, + _duplex_delta(*range(7), turn_end=True), + request, + True, + ) + + assert payload is not None + assert payload.codes.ref.tolist() == pytest.approx([0.9]) + assert payload.meta.ref_audio_sr == 16000 + + def test_full_payload_forwards_all_codes_and_request_metadata() -> None: manager = _manager() request = _request("req") - request.additional_information = { + request.model_intermediate_buffer = { "codes": {"ref": [0.1, -0.1]}, "meta": { "ref_audio_sr": 16000, diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index b7f99e3b3..e799fce81 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -42,6 +42,7 @@ from vllm_omni.metrics.modality import ( observe_audio_first_packet, observe_audio_streaming_finalize, ) +from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY from vllm_omni.model_extras import ( get_extra_body_params, get_extra_output_params, @@ -249,6 +250,44 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return True return False + async def _attach_minicpmo45_reference_audio( + self, + engine_prompt: TokPrompt, + request: ChatLikeRequest | ResponsesRequest, + ) -> None: + if not self._has_minicpmo45_stage(): + return + + reference_audio_source = getattr(request, "ref_audio", None) + extra_body = getattr(request, "extra_body", None) + model_extra = getattr(request, "model_extra", None) + if reference_audio_source is None and isinstance(extra_body, dict): + reference_audio_source = extra_body.get("ref_audio") + if reference_audio_source is None and isinstance(model_extra, dict): + nested_extra_body = model_extra.get("extra_body") + if isinstance(nested_extra_body, dict): + reference_audio_source = nested_extra_body.get("ref_audio") + if reference_audio_source is None: + reference_audio_source = model_extra.get("ref_audio") + if reference_audio_source is None: + return + if not isinstance(reference_audio_source, str): + raise ValueError("MiniCPM-o 4.5 chat ref_audio must be a URI string") + + model_config = self.model_config + media_connector = MediaConnector( + media_io_kwargs=getattr(request, "media_io_kwargs", None), + allowed_local_media_path=getattr(model_config, "allowed_local_media_path", "") or "", + allowed_media_domains=getattr(model_config, "allowed_media_domains", None), + ) + reference_waveform, reference_sample_rate = await media_connector.fetch_audio_async(reference_audio_source) + # The renderer keeps processed Stage-0 features, not the source waveform. + # Keep the decoded audio only on the original prompt for the downstream bridge. + cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY] = ( + reference_waveform, + reference_sample_rate, + ) + def _fix_minicpmo45_audio_stream_output_kinds( self, sampling_params_list: list[Any], @@ -941,6 +980,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt) prompt_additional_information["deferred_multi_modal_data"] = deferred_multi_modal_data + await self._attach_minicpmo45_reference_audio(engine_prompt, request) + speaker = getattr(request, "voice", None) or getattr(request, "speaker", None) normalized = validate_requested_speaker(speaker, self._get_supported_speakers()) if normalized is not None: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py index dfd207e7c..7017f703f 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py @@ -272,6 +272,11 @@ class MiniCPMO45Code2Wav(nn.Module): ref_audio, meta.get("ref_audio_sr"), ) + logger.debug( + "MiniCPM-o Code2Wav selected runtime reference prompt_cache_id=%s prompt_wav=%s", + entry.cache_id, + entry.path, + ) return entry.cache_id, entry.path, cache_key if previous is not None: diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py index d3d1da05b..51898f27a 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """MiniCPM-o 4.5 pipeline topology (frozen). Stage 0: Thinker — multimodal understanding + text generation. @@ -17,6 +17,7 @@ from vllm_omni.config.stage_config import ( ) _PROC = "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni" +MINICPMO45_REFERENCE_AUDIO_KEY = "_minicpmo45_reference_audio" MINICPMO_4_5_PIPELINE = PipelineConfig( diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index e303774ff..b293a95c8 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -17,6 +17,7 @@ from vllm_omni.experimental.fullduplex.engine.intermediate import ( ) from vllm_omni.inputs.data import OmniTokensPrompt from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK +from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY logger = logging.getLogger(__name__) _MINICPMO45_ASYNC_STATE = "_minicpmo45_async_codec_state" @@ -71,6 +72,23 @@ def _extract_first_audio_ref(multi_modal_data): return waveform.reshape(-1).cpu(), int(sample_rate) +def _extract_prompt_reference_audio( + prompt_item: OmniTokensPrompt | TextPrompt | None, +) -> tuple[torch.Tensor, int] | None: + if isinstance(prompt_item, Mapping): + multi_modal_data = prompt_item.get("multi_modal_data") + serving_reference_audio = prompt_item.get(MINICPMO45_REFERENCE_AUDIO_KEY) + else: + multi_modal_data = getattr(prompt_item, "multi_modal_data", None) + serving_reference_audio = getattr(prompt_item, MINICPMO45_REFERENCE_AUDIO_KEY, None) + + reference_audio = _extract_first_audio_ref(multi_modal_data) + if reference_audio is not None: + return reference_audio + + return _extract_first_audio_ref({"audio": serving_reference_audio}) + + def _extract_native_runtime_ref_audio(data_plane_metadata): if not isinstance(data_plane_metadata, dict): return None @@ -142,6 +160,17 @@ def _codec_config(transfer_manager: Any) -> tuple[int, int]: return chunk_frames, left_context_frames +def _request_intermediate_section(request: object, section: str) -> dict[str, object]: + merged: dict[str, object] = {} + for attribute in ("additional_information", "model_intermediate_buffer"): + information = getattr(request, attribute, None) + if isinstance(information, Mapping): + values = information.get(section) + if isinstance(values, Mapping): + merged.update(values) + return merged + + def _codec_scalars(value: Any) -> list[int]: """Normalize one request-routed codec delta to CPU scalar token IDs.""" if value is None: @@ -349,15 +378,13 @@ def tts2code2wav_async_chunk( ref_audio = None ref_audio_sr = None if int(record["cache_epoch"]) == 0 and chunk_seq == 0: - request_info = getattr(request, "additional_information", None) - if isinstance(request_info, Mapping): - codes_info = request_info.get("codes") - meta_info = request_info.get("meta") - raw_ref_audio = codes_info.get("ref") if isinstance(codes_info, Mapping) else None - raw_ref_audio_sr = meta_info.get("ref_audio_sr") if isinstance(meta_info, Mapping) else None - ref_audio_sr = _coerce_int(raw_ref_audio_sr) - if raw_ref_audio is not None: - ref_audio = torch.as_tensor(raw_ref_audio, dtype=torch.float32).reshape(-1).cpu() + codes_info = _request_intermediate_section(request, "codes") + meta_info = _request_intermediate_section(request, "meta") + raw_ref_audio = codes_info.get("ref") + raw_ref_audio_sr = meta_info.get("ref_audio_sr") + ref_audio_sr = _coerce_int(raw_ref_audio_sr) + if raw_ref_audio is not None: + ref_audio = torch.as_tensor(raw_ref_audio, dtype=torch.float32).reshape(-1).cpu() finished_tensor = torch.tensor(last_chunk, dtype=torch.bool) payload = OmniPayloadStruct( codes=CodesStruct( @@ -409,18 +436,9 @@ def tts2code2wav_full_payload( context = [_MINICPMO45_SILENCE_CODE] * left_context_frames if codes else [] output_codes = [*context, *codes] - request_info = getattr(request, "additional_information", None) - if not isinstance(request_info, Mapping): - request_info = {} - codes_info = request_info.get("codes") - if not isinstance(codes_info, Mapping): - codes_info = {} - meta_info = request_info.get("meta") - if not isinstance(meta_info, Mapping): - meta_info = {} - duplex_info = request_info.get("duplex") - if not isinstance(duplex_info, Mapping): - duplex_info = {} + codes_info = _request_intermediate_section(request, "codes") + meta_info = _request_intermediate_section(request, "meta") + duplex_info = _request_intermediate_section(request, "duplex") ref_audio = codes_info.get("ref") finished = torch.tensor(True, dtype=torch.bool) @@ -722,11 +740,13 @@ def llm2tts( prompt = [prompt] multi_modal_data = {} + reference_audio_by_request_id = {} for llm_output, p in zip(llm_outputs, prompt): if isinstance(p, dict): multi_modal_data[llm_output.request_id] = p.get("multi_modal_data", None) else: multi_modal_data[llm_output.request_id] = getattr(p, "multi_modal_data", None) + reference_audio_by_request_id[llm_output.request_id] = _extract_prompt_reference_audio(p) for llm_output in llm_outputs: output = llm_output.outputs[0] @@ -961,8 +981,7 @@ def llm2tts( meta["turn_start"] = native_turn_start if native_segment_end: meta["segment_end"] = True - req_mm_data = multi_modal_data.get(llm_output.request_id) - ref_audio = _extract_first_audio_ref(req_mm_data) + ref_audio = reference_audio_by_request_id.get(llm_output.request_id) if ref_audio is None: ref_audio = _extract_native_runtime_ref_audio( model_intermediate_buffer.get("duplex"),