[Bugfix] Use Seed-TTS reference audio with MiniCPM-o 4.5 (#6628)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>

tests/benchmarks/patch/test_patch.py

@@ -16,9 +16,14 @@ import pytest
 from pytest_mock import MockerFixture
 from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
 
+from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
+    SeedTTSSampleRequest,
+    SeedTTSTextSampleRequest,
+)
 from vllm_omni.benchmarks.patch.patch import (
     MixRequestFuncOutput,
     _apply_stage0_token_timings,
+    _attach_seed_tts_to_request_func_input,
     async_request_openai_chat_omni_completions,
     async_request_openai_realtime_duplex,
     should_request_stage_metrics,
@@ -28,6 +33,74 @@ from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
 pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
 
 
+def _seed_tts_request_func_input() -> RequestFuncInput:
+    return RequestFuncInput(
+        model="test-model",
+        model_name="test-model",
+        prompt="target text",
+        api_url="http://test.com/v1/chat/completions",
+        prompt_len=2,
+        output_len=20,
+        extra_body={"modalities": ["text", "audio"]},
+    )
+
+
+def test_seed_tts_chat_request_carries_reference_audio_once() -> None:
+    reference_audio_url = "data:audio/wav;base64,AAAA"
+    sample = SeedTTSSampleRequest(
+        prompt="target text",
+        prompt_len=2,
+        expected_output_len=20,
+        multi_modal_data=None,
+        seed_tts_system_prompt="Clone the supplied voice.",
+        seed_tts_speech_extra={
+            "ref_audio": reference_audio_url,
+            "ref_text": "reference text",
+            "task_type": "Base",
+        },
+    )
+    request_func_input = _seed_tts_request_func_input()
+
+    _attach_seed_tts_to_request_func_input(sample, request_func_input)
+
+    assert request_func_input.omni_chat_messages == [
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "Clone the supplied voice."}],
+        },
+        {"role": "user", "content": [{"type": "text", "text": "target text"}]},
+    ]
+    assert request_func_input.extra_body["ref_audio"] == reference_audio_url
+    serialized_request = json.dumps(
+        {"messages": request_func_input.omni_chat_messages, **request_func_input.extra_body},
+    )
+    assert serialized_request.count(reference_audio_url) == 1
+
+
+def test_seed_tts_text_chat_messages_do_not_add_reference_audio() -> None:
+    sample = SeedTTSTextSampleRequest(
+        prompt="target text",
+        prompt_len=2,
+        expected_output_len=20,
+        multi_modal_data=None,
+        seed_tts_system_prompt="Use the configured voice.",
+        seed_tts_speech_extra=None,
+    )
+    request_func_input = _seed_tts_request_func_input()
+
+    _attach_seed_tts_to_request_func_input(sample, request_func_input)
+
+    assert getattr(request_func_input, "seed_tts_row", False) is True
+    assert request_func_input.omni_chat_messages == [
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "Use the configured voice."}],
+        },
+        {"role": "user", "content": [{"type": "text", "text": "target text"}]},
+    ]
+    assert "ref_audio" not in request_func_input.extra_body
+
+
 class MockResponse:
     """Mock aiohttp response for testing"""
 

tests/benchmarks/test_seed_tts_dataset_variants.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
 and SeedTTSDesignSampleRequest.
 
@@ -234,34 +237,6 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
         assert req.seed_tts_utterance_id == "ok"
 
 
-def test_attach_sets_seed_tts_row_even_without_extra_body():
-    """seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
-    from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
-
-    req = SeedTTSTextSampleRequest(
-        prompt="hello world",
-        prompt_len=2,
-        expected_output_len=100,
-        multi_modal_data=None,
-        request_id="test-0",
-        seed_tts_speech_extra=None,
-        seed_tts_ref_wav_path="",
-    )
-    assert req.seed_tts_speech_extra is None
-    assert req.seed_tts_ref_wav_path == ""
-    # The fix ensures that even with speech_extra=None, the function
-    # sets seed_tts_row=True. We verify the source code has the fix.
-    import inspect
-
-    import vllm_omni.benchmarks.patch.patch as patch_mod
-
-    src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
-    # seed_tts_row must be set BEFORE the 'if not ex: return' check
-    row_pos = src.index("seed_tts_row")
-    not_ex_pos = src.index("if not ex:")
-    assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"
-
-
 def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
     from vllm_omni.benchmarks.data_modules import seed_tts_eval
 

tests/e2e/online_serving/test_minicpmo_4_5.py

@@ -1,12 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output.
 
 Exercises async chunk streaming (``--async-chunk``) across separate Thinker,
 Talker, and Code2Wav stages.
 """
 
+import json
 import os
+import tempfile
+from copy import deepcopy
+from pathlib import Path
 
 import pytest
+from vllm.logger import DEFAULT_LOGGING_CONFIG
 
 from tests.helpers.mark import hardware_test
 from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
@@ -23,6 +31,23 @@ _CI_DEPLOY = modify_stage_config(
     },
 )
 
+_PROMPT_LOG_TEMP_DIR = tempfile.TemporaryDirectory(prefix="minicpmo45-prompt-selection-")
+_PROMPT_SELECTION_LOG = Path(_PROMPT_LOG_TEMP_DIR.name) / "prompt-selection.log"
+_PROMPT_LOGGING_CONFIG_PATH = Path(_PROMPT_LOG_TEMP_DIR.name) / "logging.json"
+_PROMPT_LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG)
+_PROMPT_LOGGING_CONFIG["handlers"]["minicpmo_prompt_selection"] = {
+    "class": "logging.FileHandler",
+    "filename": str(_PROMPT_SELECTION_LOG),
+    "formatter": "vllm",
+    "level": "DEBUG",
+}
+_PROMPT_LOGGING_CONFIG["loggers"]["vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav"] = {
+    "handlers": ["minicpmo_prompt_selection"],
+    "level": "DEBUG",
+    "propagate": True,
+}
+_PROMPT_LOGGING_CONFIG_PATH.write_text(json.dumps(_PROMPT_LOGGING_CONFIG), encoding="utf-8")
+
 test_params = [
     pytest.param(
         OmniServerParams(
@@ -30,6 +55,10 @@ test_params = [
             stage_config_path=_CI_DEPLOY,
             use_stage_cli=False,
             server_args=["--trust-remote-code", "--async-chunk"],
+            env_dict={
+                "VLLM_CONFIGURE_LOGGING": "1",
+                "VLLM_LOGGING_CONFIG_PATH": str(_PROMPT_LOGGING_CONFIG_PATH),
+            },
         ),
         id="async_chunk",
     ),
@@ -74,6 +103,18 @@ _TTS_EXTRA_BODY = {
 }
 
 
+def _prompt_selection_log_offset() -> int:
+    return _PROMPT_SELECTION_LOG.stat().st_size if _PROMPT_SELECTION_LOG.exists() else 0
+
+
+def _read_prompt_selection_log(offset: int) -> str:
+    if not _PROMPT_SELECTION_LOG.exists():
+        return ""
+    with open(_PROMPT_SELECTION_LOG, "rb") as log_file:
+        log_file.seek(offset)
+        return log_file.read().decode("utf-8")
+
+
 @pytest.mark.core_model
 @pytest.mark.advanced_model
 @pytest.mark.omni
@@ -126,6 +167,51 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_audio_with_reference_audio(omni_server, openai_client) -> None:
+    request_config = {
+        "model": omni_server.model,
+        "messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
+        "stream": True,
+        "modalities": ["text", "audio"],
+        "extra_body": {
+            **_TTS_EXTRA_BODY,
+            "ref_audio": f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}",
+        },
+    }
+
+    log_offset = _prompt_selection_log_offset()
+    responses = openai_client.send_omni_request(request_config)
+    prompt_log = _read_prompt_selection_log(log_offset)
+
+    assert responses[0].success
+    assert responses[0].audio_bytes
+    assert "prompt_cache_id=runtime-ref-" in prompt_log
+    assert "minicpmo45_ref_" in prompt_log
+
+
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_audio_with_default_reference(omni_server, openai_client) -> None:
+    request_config = {
+        "model": omni_server.model,
+        "messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
+        "stream": True,
+        "modalities": ["text", "audio"],
+        "extra_body": _TTS_EXTRA_BODY,
+    }
+
+    responses = openai_client.send_omni_request(request_config)
+
+    assert responses[0].success
+    assert responses[0].audio_bytes
+
+
 @pytest.mark.full_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)

tests/entrypoints/openai_api/test_serving_chat_minicpmo45_reference_audio.py

@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Regression tests for MiniCPM-o 4.5 chat reference audio transport."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from types import SimpleNamespace
+from typing import cast
+
+import numpy as np
+import pytest
+import torch
+from pytest_mock import MockerFixture
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.inputs import TokensPrompt
+
+from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
+from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
+from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import (
+    llm2tts,
+    tts2code2wav_async_chunk,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+@dataclass
+class _EngineClientStub:
+    stage_configs: list[dict[str, object]]
+
+
+@dataclass
+class _ModelConfigStub:
+    allowed_local_media_path: str = ""
+    allowed_media_domains: list[str] | None = None
+
+
+@dataclass
+class _ThinkerCompletionStub:
+    token_ids: list[int]
+    text: str
+    hidden_states: torch.Tensor
+    multimodal_output: dict[str, object] = field(default_factory=dict)
+
+
+@dataclass
+class _ThinkerRequestOutputStub:
+    request_id: str
+    prompt_token_ids: list[int]
+    outputs: list[_ThinkerCompletionStub]
+    multimodal_output: dict[str, object] | None = None
+
+
+@dataclass
+class _Code2WavRequestStub:
+    external_req_id: str
+    request_id: str
+    model_intermediate_buffer: dict[str, object]
+
+
+@dataclass
+class _TransferManagerStub:
+    request_payload: dict[str, object] = field(default_factory=dict)
+
+
+@pytest.fixture
+def serving_chat() -> OmniOpenAIServingChat:
+    return object.__new__(OmniOpenAIServingChat)
+
+
+def _stage_config(model_arch: str) -> dict[str, object]:
+    return {"engine_args": {"model_arch": model_arch}}
+
+
+def _chat_request(reference_audio_url: str) -> ChatCompletionRequest:
+    return ChatCompletionRequest(
+        model="test-model",
+        messages=[{"role": "user", "content": "target text"}],
+        ref_audio=reference_audio_url,
+    )
+
+
+@pytest.mark.asyncio
+async def test_chat_reference_audio_reaches_code2wav_transport(
+    serving_chat: OmniOpenAIServingChat,
+    mocker: MockerFixture,
+) -> None:
+    reference_audio_url = "data:audio/wav;base64,AAAA"
+    reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
+    serving_chat.engine_client = _EngineClientStub(
+        [_stage_config("MiniCPMO45OmniForConditionalGeneration")],
+    )
+    serving_chat.model_config = _ModelConfigStub()
+    engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
+    media_connector_class = mocker.patch(
+        "vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
+    )
+    media_connector = media_connector_class.return_value
+    media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
+
+    await serving_chat._attach_minicpmo45_reference_audio(
+        engine_prompt,
+        _chat_request(reference_audio_url),
+    )
+
+    media_connector_class.assert_called_once_with(
+        media_io_kwargs=None,
+        allowed_local_media_path="",
+        allowed_media_domains=None,
+    )
+    media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
+    preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY]
+    assert isinstance(preserved_reference, tuple)
+    preserved_waveform, preserved_sample_rate = preserved_reference
+    assert isinstance(preserved_waveform, np.ndarray)
+    assert isinstance(preserved_sample_rate, int)
+    np.testing.assert_array_equal(preserved_waveform, reference_waveform)
+    assert preserved_sample_rate == 16000
+
+    thinker_output = _ThinkerRequestOutputStub(
+        request_id="request-0",
+        prompt_token_ids=[10],
+        outputs=[
+            _ThinkerCompletionStub(
+                token_ids=[20],
+                text="target text",
+                hidden_states=torch.zeros((2, 4)),
+            )
+        ],
+    )
+    talker_prompt = llm2tts([thinker_output], prompt=engine_prompt)[0]
+    talker_request = _Code2WavRequestStub(
+        external_req_id="request-0",
+        request_id="request-0",
+        model_intermediate_buffer=talker_prompt["model_intermediate_buffer"],
+    )
+    transfer_manager = _TransferManagerStub()
+
+    code2wav_payload = tts2code2wav_async_chunk(
+        transfer_manager,
+        {"codes": {"audio": torch.arange(7).reshape(-1, 1)}},
+        talker_request,
+        True,
+    )
+
+    assert code2wav_payload is not None
+    torch.testing.assert_close(code2wav_payload.codes.ref, torch.from_numpy(reference_waveform))
+    assert code2wav_payload.meta.ref_audio_sr == 16000
+
+
+@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"])
+@pytest.mark.asyncio
+async def test_reference_audio_accepts_request_extra_mappings(
+    serving_chat: OmniOpenAIServingChat,
+    mocker: MockerFixture,
+    request_shape: str,
+) -> None:
+    reference_audio_url = "data:audio/wav;base64,AAAA"
+    reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
+    serving_chat.engine_client = _EngineClientStub(
+        [_stage_config("MiniCPMO45OmniForConditionalGeneration")],
+    )
+    serving_chat.model_config = _ModelConfigStub()
+    engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
+    if request_shape == "extra_body":
+        request = ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "target text"}],
+            extra_body={"ref_audio": reference_audio_url},
+        )
+    else:
+        request = SimpleNamespace(model_extra={"ref_audio": reference_audio_url})
+    assert not hasattr(request, "ref_audio")
+    media_connector = mocker.patch(
+        "vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
+    ).return_value
+    media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
+
+    await serving_chat._attach_minicpmo45_reference_audio(engine_prompt, request)
+
+    media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
+    preserved_reference = cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY]
+    assert isinstance(preserved_reference, tuple)
+    np.testing.assert_array_equal(preserved_reference[0], reference_waveform)
+    assert preserved_reference[1] == 16000
+
+
+@pytest.mark.asyncio
+async def test_reference_audio_is_ignored_for_other_models(
+    serving_chat: OmniOpenAIServingChat,
+    mocker: MockerFixture,
+) -> None:
+    reference_audio_url = "data:audio/wav;base64,AAAA"
+    serving_chat.engine_client = _EngineClientStub([_stage_config("SomeOtherOmniForConditionalGeneration")])
+    engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
+    media_connector_class = mocker.patch(
+        "vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
+    )
+
+    await serving_chat._attach_minicpmo45_reference_audio(
+        engine_prompt,
+        _chat_request(reference_audio_url),
+    )
+
+    media_connector_class.assert_not_called()
+    assert MINICPMO45_REFERENCE_AUDIO_KEY not in cast(dict[str, object], engine_prompt)

tests/model_executor/models/minicpmo_4_5/test_llm2tts.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Unit tests for the MiniCPM-o 4.5 stage 0 -> stage 1 bridge.
 
 Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts``:
@@ -25,6 +25,7 @@ import torch
 from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
     MiniCPMO45OmniForConditionalGeneration,
 )
+from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
 from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -313,6 +314,44 @@ class TestPromptAndMultiModal:
         )
         assert out[0]["multi_modal_data"] == mm
 
+    def test_reference_audio_is_added_to_stage_handoff(self) -> None:
+        hidden_states = torch.zeros((2, _HIDDEN_DIM))
+        waveform = torch.tensor([0.25, -0.5], dtype=torch.float32)
+
+        stage_outputs = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
+            prompt={"multi_modal_data": {"audio": (waveform, 16000)}},
+        )
+
+        intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
+        assert intermediate_buffer["codes"]["ref"] == [0.25, -0.5]
+        assert intermediate_buffer["meta"]["ref_audio_sr"] == 16000
+
+    def test_serving_reference_audio_is_added_to_stage_handoff(self) -> None:
+        hidden_states = torch.zeros((2, _HIDDEN_DIM))
+        waveform = torch.tensor([0.125, -0.25], dtype=torch.float32)
+
+        stage_outputs = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
+            prompt={MINICPMO45_REFERENCE_AUDIO_KEY: (waveform, 24000)},
+        )
+
+        intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
+        assert intermediate_buffer["codes"]["ref"] == [0.125, -0.25]
+        assert intermediate_buffer["meta"]["ref_audio_sr"] == 24000
+
+    def test_missing_reference_audio_does_not_add_stage_handoff_fields(self) -> None:
+        hidden_states = torch.zeros((2, _HIDDEN_DIM))
+
+        stage_outputs = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden_states)],
+            prompt={"multi_modal_data": {}},
+        )
+
+        intermediate_buffer = stage_outputs[0]["model_intermediate_buffer"]
+        assert "codes" not in intermediate_buffer
+        assert "ref_audio_sr" not in intermediate_buffer.get("meta", {})
+
     def test_internal_streaming_context_is_accepted(self) -> None:
         hidden = torch.zeros((2, _HIDDEN_DIM))
         out = llm2tts(

tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 from collections import defaultdict
 from types import SimpleNamespace
@@ -168,10 +168,14 @@ def test_duplex_turn_end_waits_for_terminal_codec_flush() -> None:
 def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None:
     manager = _manager()
     request = _request("req")
-    request.additional_information = {
+    request.model_intermediate_buffer = {
         "codes": {"ref": [0.1, -0.1]},
         "meta": {"ref_audio_sr": 16000},
     }
+    request.additional_information = {
+        "codes": {"ref": [0.9]},
+        "meta": {"ref_audio_sr": 8000},
+    }
 
     payload = tts2code2wav_async_chunk(
         manager,
@@ -193,10 +197,33 @@ def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None:
     assert payload.meta.turn_end is True
 
 
+def test_first_chunk_falls_back_to_legacy_reference_fields() -> None:
+    manager = _manager()
+    request = _request("req")
+    request.model_intermediate_buffer = {
+        "meta": {"ref_audio_sr": 16000},
+    }
+    request.additional_information = {
+        "codes": {"ref": [0.9]},
+        "meta": {"ref_audio_sr": 8000},
+    }
+
+    payload = tts2code2wav_async_chunk(
+        manager,
+        _duplex_delta(*range(7), turn_end=True),
+        request,
+        True,
+    )
+
+    assert payload is not None
+    assert payload.codes.ref.tolist() == pytest.approx([0.9])
+    assert payload.meta.ref_audio_sr == 16000
+
+
 def test_full_payload_forwards_all_codes_and_request_metadata() -> None:
     manager = _manager()
     request = _request("req")
-    request.additional_information = {
+    request.model_intermediate_buffer = {
         "codes": {"ref": [0.1, -0.1]},
         "meta": {
             "ref_audio_sr": 16000,

vllm_omni/entrypoints/openai/serving_chat.py

@@ -42,6 +42,7 @@ from vllm_omni.metrics.modality import (
     observe_audio_first_packet,
     observe_audio_streaming_finalize,
 )
+from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
 from vllm_omni.model_extras import (
     get_extra_body_params,
     get_extra_output_params,
@@ -249,6 +250,44 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
                 return True
         return False
 
+    async def _attach_minicpmo45_reference_audio(
+        self,
+        engine_prompt: TokPrompt,
+        request: ChatLikeRequest | ResponsesRequest,
+    ) -> None:
+        if not self._has_minicpmo45_stage():
+            return
+
+        reference_audio_source = getattr(request, "ref_audio", None)
+        extra_body = getattr(request, "extra_body", None)
+        model_extra = getattr(request, "model_extra", None)
+        if reference_audio_source is None and isinstance(extra_body, dict):
+            reference_audio_source = extra_body.get("ref_audio")
+        if reference_audio_source is None and isinstance(model_extra, dict):
+            nested_extra_body = model_extra.get("extra_body")
+            if isinstance(nested_extra_body, dict):
+                reference_audio_source = nested_extra_body.get("ref_audio")
+            if reference_audio_source is None:
+                reference_audio_source = model_extra.get("ref_audio")
+        if reference_audio_source is None:
+            return
+        if not isinstance(reference_audio_source, str):
+            raise ValueError("MiniCPM-o 4.5 chat ref_audio must be a URI string")
+
+        model_config = self.model_config
+        media_connector = MediaConnector(
+            media_io_kwargs=getattr(request, "media_io_kwargs", None),
+            allowed_local_media_path=getattr(model_config, "allowed_local_media_path", "") or "",
+            allowed_media_domains=getattr(model_config, "allowed_media_domains", None),
+        )
+        reference_waveform, reference_sample_rate = await media_connector.fetch_audio_async(reference_audio_source)
+        # The renderer keeps processed Stage-0 features, not the source waveform.
+        # Keep the decoded audio only on the original prompt for the downstream bridge.
+        cast(dict[str, object], engine_prompt)[MINICPMO45_REFERENCE_AUDIO_KEY] = (
+            reference_waveform,
+            reference_sample_rate,
+        )
+
     def _fix_minicpmo45_audio_stream_output_kinds(
         self,
         sampling_params_list: list[Any],
@@ -941,6 +980,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
             prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
             prompt_additional_information["deferred_multi_modal_data"] = deferred_multi_modal_data
 
+        await self._attach_minicpmo45_reference_audio(engine_prompt, request)
+
         speaker = getattr(request, "voice", None) or getattr(request, "speaker", None)
         normalized = validate_requested_speaker(speaker, self._get_supported_speakers())
         if normalized is not None:

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py

@@ -272,6 +272,11 @@ class MiniCPMO45Code2Wav(nn.Module):
                 ref_audio,
                 meta.get("ref_audio_sr"),
             )
+            logger.debug(
+                "MiniCPM-o Code2Wav selected runtime reference prompt_cache_id=%s prompt_wav=%s",
+                entry.cache_id,
+                entry.path,
+            )
             return entry.cache_id, entry.path, cache_key
 
         if previous is not None:

vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """MiniCPM-o 4.5 pipeline topology (frozen).
 
 Stage 0: Thinker — multimodal understanding + text generation.
@@ -17,6 +17,7 @@ from vllm_omni.config.stage_config import (
 )
 
 _PROC = "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni"
+MINICPMO45_REFERENCE_AUDIO_KEY = "_minicpmo45_reference_audio"
 
 
 MINICPMO_4_5_PIPELINE = PipelineConfig(

vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py

@@ -17,6 +17,7 @@ from vllm_omni.experimental.fullduplex.engine.intermediate import (
 )
 from vllm_omni.inputs.data import OmniTokensPrompt
 from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
+from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
 
 logger = logging.getLogger(__name__)
 _MINICPMO45_ASYNC_STATE = "_minicpmo45_async_codec_state"
@@ -71,6 +72,23 @@ def _extract_first_audio_ref(multi_modal_data):
     return waveform.reshape(-1).cpu(), int(sample_rate)
 
 
+def _extract_prompt_reference_audio(
+    prompt_item: OmniTokensPrompt | TextPrompt | None,
+) -> tuple[torch.Tensor, int] | None:
+    if isinstance(prompt_item, Mapping):
+        multi_modal_data = prompt_item.get("multi_modal_data")
+        serving_reference_audio = prompt_item.get(MINICPMO45_REFERENCE_AUDIO_KEY)
+    else:
+        multi_modal_data = getattr(prompt_item, "multi_modal_data", None)
+        serving_reference_audio = getattr(prompt_item, MINICPMO45_REFERENCE_AUDIO_KEY, None)
+
+    reference_audio = _extract_first_audio_ref(multi_modal_data)
+    if reference_audio is not None:
+        return reference_audio
+
+    return _extract_first_audio_ref({"audio": serving_reference_audio})
+
+
 def _extract_native_runtime_ref_audio(data_plane_metadata):
     if not isinstance(data_plane_metadata, dict):
         return None
@@ -142,6 +160,17 @@ def _codec_config(transfer_manager: Any) -> tuple[int, int]:
     return chunk_frames, left_context_frames
 
 
+def _request_intermediate_section(request: object, section: str) -> dict[str, object]:
+    merged: dict[str, object] = {}
+    for attribute in ("additional_information", "model_intermediate_buffer"):
+        information = getattr(request, attribute, None)
+        if isinstance(information, Mapping):
+            values = information.get(section)
+            if isinstance(values, Mapping):
+                merged.update(values)
+    return merged
+
+
 def _codec_scalars(value: Any) -> list[int]:
     """Normalize one request-routed codec delta to CPU scalar token IDs."""
     if value is None:
@@ -349,15 +378,13 @@ def tts2code2wav_async_chunk(
     ref_audio = None
     ref_audio_sr = None
     if int(record["cache_epoch"]) == 0 and chunk_seq == 0:
-        request_info = getattr(request, "additional_information", None)
-        if isinstance(request_info, Mapping):
-            codes_info = request_info.get("codes")
-            meta_info = request_info.get("meta")
-            raw_ref_audio = codes_info.get("ref") if isinstance(codes_info, Mapping) else None
-            raw_ref_audio_sr = meta_info.get("ref_audio_sr") if isinstance(meta_info, Mapping) else None
-            ref_audio_sr = _coerce_int(raw_ref_audio_sr)
-            if raw_ref_audio is not None:
-                ref_audio = torch.as_tensor(raw_ref_audio, dtype=torch.float32).reshape(-1).cpu()
+        codes_info = _request_intermediate_section(request, "codes")
+        meta_info = _request_intermediate_section(request, "meta")
+        raw_ref_audio = codes_info.get("ref")
+        raw_ref_audio_sr = meta_info.get("ref_audio_sr")
+        ref_audio_sr = _coerce_int(raw_ref_audio_sr)
+        if raw_ref_audio is not None:
+            ref_audio = torch.as_tensor(raw_ref_audio, dtype=torch.float32).reshape(-1).cpu()
     finished_tensor = torch.tensor(last_chunk, dtype=torch.bool)
     payload = OmniPayloadStruct(
         codes=CodesStruct(
@@ -409,18 +436,9 @@ def tts2code2wav_full_payload(
     context = [_MINICPMO45_SILENCE_CODE] * left_context_frames if codes else []
     output_codes = [*context, *codes]
 
-    request_info = getattr(request, "additional_information", None)
-    if not isinstance(request_info, Mapping):
-        request_info = {}
-    codes_info = request_info.get("codes")
-    if not isinstance(codes_info, Mapping):
-        codes_info = {}
-    meta_info = request_info.get("meta")
-    if not isinstance(meta_info, Mapping):
-        meta_info = {}
-    duplex_info = request_info.get("duplex")
-    if not isinstance(duplex_info, Mapping):
-        duplex_info = {}
+    codes_info = _request_intermediate_section(request, "codes")
+    meta_info = _request_intermediate_section(request, "meta")
+    duplex_info = _request_intermediate_section(request, "duplex")
 
     ref_audio = codes_info.get("ref")
     finished = torch.tensor(True, dtype=torch.bool)
@@ -722,11 +740,13 @@ def llm2tts(
         prompt = [prompt]
 
     multi_modal_data = {}
+    reference_audio_by_request_id = {}
     for llm_output, p in zip(llm_outputs, prompt):
         if isinstance(p, dict):
             multi_modal_data[llm_output.request_id] = p.get("multi_modal_data", None)
         else:
             multi_modal_data[llm_output.request_id] = getattr(p, "multi_modal_data", None)
+        reference_audio_by_request_id[llm_output.request_id] = _extract_prompt_reference_audio(p)
 
     for llm_output in llm_outputs:
         output = llm_output.outputs[0]
@@ -961,8 +981,7 @@ def llm2tts(
             meta["turn_start"] = native_turn_start
             if native_segment_end:
                 meta["segment_end"] = True
-        req_mm_data = multi_modal_data.get(llm_output.request_id)
-        ref_audio = _extract_first_audio_ref(req_mm_data)
+        ref_audio = reference_audio_by_request_id.get(llm_output.request_id)
         if ref_audio is None:
             ref_audio = _extract_native_runtime_ref_audio(
                 model_intermediate_buffer.get("duplex"),