[Full Duplex] Feat: Support Full-Duplex realtime runtime & add MiniCPM-o 4.5 demo (#3907)

Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: StormMelody <22371184@zju.edu.cn>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-07-23 16:45:18 +08:00
1 parent b55120c93b
commit 05b794f29b
142 files changed
+48389 -1013

No files matched your search

@@ -5,7 +5,6 @@
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
MiniCPMO45OmniTTSForConditionalGeneration,
@@ -14,10 +13,19 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _FakeMiniCPMTTS:
def __init__(self, config, audio_tokenizer):
self.config = config
self.audio_tokenizer = audio_tokenizer
self.emb_text = object()
self.projector_semantic = object()
def generate(self):
raise NotImplementedError
def test_hf_model_id_path_caching(mocker, tmp_path):
"""Verify HF model-ID path is resolved via download_weights_from_hf_specific
once during _lazy_init_tts, cached in self._model_path, and reused in generate_speech.
"""
"""Verify the resolved HF path is cached and reused for default ref audio."""
resolved_path = str(tmp_path / "resolved_hf_model")
assets_dir = tmp_path / "resolved_hf_model" / "assets"
assets_dir.mkdir(parents=True, exist_ok=True)
@@ -29,11 +37,9 @@ def test_hf_model_id_path_caching(mocker, tmp_path):
return_value=resolved_path,
)
# Mock dynamic class loading
fake_tts_class = mocker.MagicMock()
mocker.patch(
"transformers.dynamic_module_utils.get_class_from_dynamic_module",
return_value=fake_tts_class,
return_value=_FakeMiniCPMTTS,
)
# Build fake vllm_config with HF model ID
@@ -59,40 +65,10 @@ def test_hf_model_id_path_caching(mocker, tmp_path):
tts_model = MiniCPMO45OmniTTSForConditionalGeneration(vllm_config=vllm_config)
# 1. Trigger lazy init
tts_model._lazy_init_tts()
tts_model._lazy_init_tts()
# Verify download_weights_from_hf_specific was called with HF model ID and cached
mock_download.assert_called_once_with(model_id, None, ["*"])
assert tts_model._model_path == resolved_path
assert tts_model._assets_loaded is True
# 2. Mock TTS object generate method and audio tokenizer for generate_speech
fake_tts_obj = mocker.MagicMock()
fake_tts_obj.emb_text.weight.device = torch.device("cpu")
fake_tts_obj.emb_text.side_effect = lambda t: torch.zeros(t.shape[0], 768)
fake_tts_obj.projector_semantic.side_effect = lambda h, **kw: torch.zeros(h.shape[0], 768)
fake_tts_obj.config = tts_cfg
fake_tts_obj.audio_bos_token_id = 151687
fake_output = SimpleNamespace(new_ids=torch.tensor([[[100]]]))
fake_tts_obj.generate.return_value = fake_output
tts_model.tts_obj = fake_tts_obj
fake_tokenizer = mocker.MagicMock(return_value=b"dummy_wav_bytes")
tts_model.audio_tokenizer = fake_tokenizer
# Mock sf.read to return fake numpy audio
mocker.patch(
"soundfile.read",
return_value=(torch.zeros(16000).numpy(), 24000),
)
# Call generate_speech
tts_token_ids = torch.tensor([1, 2, 3])
tts_hidden_states = torch.zeros(3, 768)
waveform = tts_model.generate_speech(tts_token_ids, tts_hidden_states)
# Verify download_weights_from_hf_specific was NOT called a second time
assert mock_download.call_count == 1
assert waveform is not None
assert tts_model._resolve_prompt_wav_path(None, None) == (str(ref_audio_file), None)
@@ -7,11 +7,9 @@ Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2t
- empty ``source_outputs`` raises
- latent fallback to ``hidden_states`` when ``multimodal_output`` is empty
- both inputs missing -> raises
- additional_information payload carries the keys the talker expects
(prompt_embeds, prompt_token_ids, llm_output_token_ids, llm_output_text)
- dummy talker prompt ``[BOS, PAD, EOS] = [1, 0, 2]`` (single prefill step)
- structured model_intermediate_buffer carries the thinker ids and text
- scheduler prompt tokens follow the selected TTS region or output tokens
- MiniCPM-o 4.5 TTS region detection on 151703 / 151704 tokens
- MiniCPM-o 2.6 fallback detection on 151691 / 151692 when no 4.5 markers
- No TTS markers present -> no ``tts_token_ids`` / ``tts_hidden_states`` keys
- prompt arg is normalized to a list and ``multi_modal_data`` is gated by
``requires_multimodal_data``
@@ -24,6 +22,9 @@ from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
MiniCPMO45OmniForConditionalGeneration,
)
from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -32,6 +33,55 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_HIDDEN_DIM = 4
def test_native_duplex_talker_emits_empty_text_metadata_to_clear_previous_segment() -> None:
class _Talker:
_ar_last_chunk_flags = [True]
_ar_turn_end_flags = [True]
def __call__(self, **kwargs):
del kwargs
return None, torch.zeros(0, dtype=torch.float32)
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
torch.nn.Module.__init__(model)
model.model_stage = "tts"
model.config = SimpleNamespace(hidden_size=_HIDDEN_DIM)
model.talker = _Talker()
output = model.forward(
input_ids=torch.tensor([1]),
positions=torch.tensor([0]),
runtime_additional_information=[
{
"native_duplex": True,
"duplex": {"turn_id": 1, "epoch": 0},
"meta": {"native_duplex_segment_text": ""},
}
],
)
assert output.multimodal_outputs is not None
assert output.multimodal_outputs["meta.llm_output_text_utf8"].numel() == 0
assert output.multimodal_outputs["meta.audio_text_total_chars"].tolist() == [0]
def test_tts_embed_input_ids_uses_active_talker_embedding(monkeypatch) -> None:
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
torch.nn.Module.__init__(model)
model.model_stage = "tts"
expected = torch.ones((2, _HIDDEN_DIM))
monkeypatch.setattr(
model,
"get_input_embeddings",
lambda input_ids: expected,
)
actual = model.embed_input_ids(torch.tensor([11, 12]))
assert actual is expected
def _make_thinker_output(
*,
prompt_token_ids: list[int],
@@ -40,6 +90,7 @@ def _make_thinker_output(
request_id: str = "req-0",
latent: torch.Tensor | None = None,
hidden_states: torch.Tensor | None = None,
multimodal_output: dict[str, object] | None = None,
):
"""Construct a minimal mock of a thinker engine output entry.
@@ -48,8 +99,11 @@ def _make_thinker_output(
- per-output: ``multimodal_output`` (dict), ``hidden_states`` (opt),
``text``, ``token_ids``
"""
mm_output = dict(multimodal_output or {})
if latent is not None:
mm_output["latent"] = latent
output = SimpleNamespace(
multimodal_output={"latent": latent} if latent is not None else {},
multimodal_output=mm_output,
token_ids=output_token_ids,
text=text,
)
@@ -90,17 +144,16 @@ class TestBasicShape:
)
assert len(out) == 2
def test_talker_prompt_token_ids_dummy_bos_pad_eos(self) -> None:
def test_talker_scheduler_prompt_uses_output_tokens(self) -> None:
hidden = torch.zeros((2, _HIDDEN_DIM))
out = llm2tts(
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
prompt=None,
)
# The talker AR framework needs *some* tokens to do a single prefill;
# this is the agreed [BOS, PAD, EOS] minimal payload.
assert out[0]["prompt_token_ids"] == [1, 0, 2]
assert out[0]["prompt_token_ids"] == [20]
assert "stream_output" not in out[0]["model_intermediate_buffer"]
def test_additional_information_carries_thinker_outputs(self) -> None:
def test_model_intermediate_buffer_carries_thinker_outputs(self) -> None:
prompt_ids = [10, 11, 12]
out_ids = [20, 21]
hidden = torch.randn(len(prompt_ids) + len(out_ids), _HIDDEN_DIM)
@@ -116,21 +169,17 @@ class TestBasicShape:
],
prompt=None,
)
ai = result[0]["additional_information"]
assert ai["prompt_token_ids"] == prompt_ids
assert ai["llm_output_token_ids"] == out_ids
assert ai["llm_output_text"] == ["hello"]
# prompt_embeds = float32 view of the prompt-portion of hidden states.
assert ai["prompt_embeds"].dtype == torch.float32
assert ai["prompt_embeds"].shape == (len(prompt_ids), _HIDDEN_DIM)
assert torch.equal(ai["prompt_embeds"], hidden[: len(prompt_ids)].to(torch.float32))
buffer = result[0]["model_intermediate_buffer"]
assert buffer["ids"]["prompt"] == prompt_ids
assert buffer["ids"]["output"] == out_ids
assert buffer["llm_output_text"] == ["hello"]
def test_latent_in_multimodal_output_takes_precedence(self) -> None:
# When both ``multimodal_output["latent"]`` and ``hidden_states`` are
# present, the latent payload must win (this is the steady-state path
# produced by the thinker stage).
prompt_ids = [10, 11]
out_ids = [20]
out_ids = [151703, 30, 151704]
latent = torch.ones((len(prompt_ids) + len(out_ids), _HIDDEN_DIM))
hidden = torch.zeros_like(latent)
@@ -145,19 +194,17 @@ class TestBasicShape:
],
prompt=None,
)
ai = result[0]["additional_information"]
buffer = result[0]["model_intermediate_buffer"]
# latent (ones) won over hidden_states (zeros)
assert torch.equal(ai["prompt_embeds"], latent[: len(prompt_ids)].to(torch.float32))
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), latent[3:4].to(torch.float32))
class TestTtsRegionDetection:
"""The bridge auto-detects which MiniCPM-o generation produced the output
by sniffing for 4.5-specific BOS/EOS token ids (151703 / 151704); when
those are absent it falls back to the 2.6 ids (151691 / 151692).
"""The bridge detects MiniCPM-o 4.5 BOS/EOS ids (151703 / 151704).
This is a regression guard: a single off-by-one or wrong branch
precedence here causes the talker to receive an empty / wrong-region
slice and emit silent audio.
causes the talker to receive an empty / wrong-region slice and emit
silent audio.
"""
def _run(self, prompt_token_ids, output_token_ids):
@@ -173,7 +220,7 @@ class TestTtsRegionDetection:
],
prompt=None,
)
return result[0]["additional_information"], hidden
return result[0]["model_intermediate_buffer"], hidden
def test_4_5_markers_detected(self) -> None:
# prompt: [10, 11]
@@ -181,45 +228,26 @@ class TestTtsRegionDetection:
# full sequence: [10, 11, 151703, 30, 31, 151704, 40]
# 0 1 2 3 4 5 6
# 4.5 BOS at idx 2 -> slice starts at 3; EOS at idx 5 -> slice ends at 5.
ai, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40])
assert "tts_token_ids" in ai and "tts_hidden_states" in ai
assert ai["tts_token_ids"].tolist() == [30, 31]
assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
def test_4_5_takes_precedence_when_both_markers_present(self) -> None:
# If both 2.6 and 4.5 markers appear (shouldn't really happen on a
# real checkpoint, but exercises the loop's break), the 4.5 markers
# win because the source loop pins on the first 4.5 id seen.
# sequence index: 0 1 2 3 4 5 6 7 8
ai, hidden = self._run([10], [151691, 70, 151692, 151703, 30, 151704, 40])
assert ai["tts_token_ids"].tolist() == [30]
assert torch.equal(ai["tts_hidden_states"], hidden[5:6])
def test_2_6_fallback(self) -> None:
# No 4.5 markers anywhere -> use 2.6 BOS/EOS pair.
# full sequence: [10, 151691, 30, 31, 151692, 40]
# 0 1 2 3 4 5
# BOS at idx 1 -> slice starts at 2; EOS at idx 4 -> slice ends at 4.
ai, hidden = self._run([10], [151691, 30, 31, 151692, 40])
assert ai["tts_token_ids"].tolist() == [30, 31]
assert torch.equal(ai["tts_hidden_states"], hidden[2:4])
buffer, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40])
assert buffer["ids"]["tts"] == [30, 31]
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5])
def test_bos_without_eos_runs_to_end(self) -> None:
# When BOS is found but EOS is missing (typical for an in-flight or
# truncated decode), the slice should extend to the end of the
# hidden-state matrix instead of being dropped.
# sequence: [10, 11, 151703, 30, 31]
ai, hidden = self._run([10, 11], [151703, 30, 31])
assert ai["tts_token_ids"].tolist() == [30, 31]
assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
buffer, hidden = self._run([10, 11], [151703, 30, 31])
assert buffer["ids"]["tts"] == [30, 31]
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5])
def test_no_tts_markers_omits_slice_keys(self) -> None:
# If neither marker pair is present, the bridge should NOT populate
# ``tts_token_ids`` / ``tts_hidden_states`` — the talker should fall
# through to the dummy path.
ai, _ = self._run([10, 11], [20, 21, 22])
assert "tts_token_ids" not in ai
assert "tts_hidden_states" not in ai
buffer, _ = self._run([10, 11], [20, 21, 22])
assert "tts" not in buffer["ids"]
assert "tts" not in buffer.get("hidden_states", {})
class TestPromptAndMultiModal:
@@ -251,14 +279,11 @@ class TestPromptAndMultiModal:
)
assert out[0]["multi_modal_data"] == mm
def test_streaming_context_is_accepted_and_ignored(self) -> None:
# The framework passes ``streaming_context`` for streaming-capable
# pipelines; MiniCPM-o 4.5 is turn-taking so the bridge must accept
# but discard it without crashing.
def test_internal_streaming_context_is_accepted(self) -> None:
hidden = torch.zeros((2, _HIDDEN_DIM))
out = llm2tts(
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
prompt=None,
streaming_context=object(),
_streaming_context=object(),
)
assert len(out) == 1
@@ -44,6 +44,20 @@ class TestRegistryDeclaration:
assert pipeline.model_type == _PIPELINE_KEY
assert pipeline.model_arch == "MiniCPMO45OmniForConditionalGeneration"
def test_native_duplex_control_is_explicit_without_a_fixed_session_cap(self) -> None:
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
assert pipeline.duplex_control_enabled is True
assert pipeline.duplex_serving_adapter == (
"vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
)
assert not hasattr(pipeline, "max_native_duplex_sessions")
def test_ordinary_pipeline_defaults_to_no_duplex_control(self) -> None:
pipeline = PipelineConfig(model_type="ordinary")
assert pipeline.duplex_control_enabled is False
assert pipeline.duplex_serving_adapter is None
assert not hasattr(pipeline, "max_native_duplex_sessions")
class TestPipelineTopology:
@pytest.fixture(scope="class")