[Full Duplex] Feat: Support Full-Duplex realtime runtime & add MiniCPM-o 4.5 demo (#3907)
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: StormMelody <22371184@zju.edu.cn> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
142 files changed
+48389
-1013
No files matched your search
@@ -5,7 +5,6 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
@@ -14,10 +13,19 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _FakeMiniCPMTTS:
|
||||
def __init__(self, config, audio_tokenizer):
|
||||
self.config = config
|
||||
self.audio_tokenizer = audio_tokenizer
|
||||
self.emb_text = object()
|
||||
self.projector_semantic = object()
|
||||
|
||||
def generate(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def test_hf_model_id_path_caching(mocker, tmp_path):
|
||||
"""Verify HF model-ID path is resolved via download_weights_from_hf_specific
|
||||
once during _lazy_init_tts, cached in self._model_path, and reused in generate_speech.
|
||||
"""
|
||||
"""Verify the resolved HF path is cached and reused for default ref audio."""
|
||||
resolved_path = str(tmp_path / "resolved_hf_model")
|
||||
assets_dir = tmp_path / "resolved_hf_model" / "assets"
|
||||
assets_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -29,11 +37,9 @@ def test_hf_model_id_path_caching(mocker, tmp_path):
|
||||
return_value=resolved_path,
|
||||
)
|
||||
|
||||
# Mock dynamic class loading
|
||||
fake_tts_class = mocker.MagicMock()
|
||||
mocker.patch(
|
||||
"transformers.dynamic_module_utils.get_class_from_dynamic_module",
|
||||
return_value=fake_tts_class,
|
||||
return_value=_FakeMiniCPMTTS,
|
||||
)
|
||||
|
||||
# Build fake vllm_config with HF model ID
|
||||
@@ -59,40 +65,10 @@ def test_hf_model_id_path_caching(mocker, tmp_path):
|
||||
|
||||
tts_model = MiniCPMO45OmniTTSForConditionalGeneration(vllm_config=vllm_config)
|
||||
|
||||
# 1. Trigger lazy init
|
||||
tts_model._lazy_init_tts()
|
||||
tts_model._lazy_init_tts()
|
||||
|
||||
# Verify download_weights_from_hf_specific was called with HF model ID and cached
|
||||
mock_download.assert_called_once_with(model_id, None, ["*"])
|
||||
assert tts_model._model_path == resolved_path
|
||||
assert tts_model._assets_loaded is True
|
||||
|
||||
# 2. Mock TTS object generate method and audio tokenizer for generate_speech
|
||||
fake_tts_obj = mocker.MagicMock()
|
||||
fake_tts_obj.emb_text.weight.device = torch.device("cpu")
|
||||
fake_tts_obj.emb_text.side_effect = lambda t: torch.zeros(t.shape[0], 768)
|
||||
fake_tts_obj.projector_semantic.side_effect = lambda h, **kw: torch.zeros(h.shape[0], 768)
|
||||
fake_tts_obj.config = tts_cfg
|
||||
fake_tts_obj.audio_bos_token_id = 151687
|
||||
|
||||
fake_output = SimpleNamespace(new_ids=torch.tensor([[[100]]]))
|
||||
fake_tts_obj.generate.return_value = fake_output
|
||||
|
||||
tts_model.tts_obj = fake_tts_obj
|
||||
fake_tokenizer = mocker.MagicMock(return_value=b"dummy_wav_bytes")
|
||||
tts_model.audio_tokenizer = fake_tokenizer
|
||||
|
||||
# Mock sf.read to return fake numpy audio
|
||||
mocker.patch(
|
||||
"soundfile.read",
|
||||
return_value=(torch.zeros(16000).numpy(), 24000),
|
||||
)
|
||||
|
||||
# Call generate_speech
|
||||
tts_token_ids = torch.tensor([1, 2, 3])
|
||||
tts_hidden_states = torch.zeros(3, 768)
|
||||
waveform = tts_model.generate_speech(tts_token_ids, tts_hidden_states)
|
||||
|
||||
# Verify download_weights_from_hf_specific was NOT called a second time
|
||||
assert mock_download.call_count == 1
|
||||
assert waveform is not None
|
||||
assert tts_model._resolve_prompt_wav_path(None, None) == (str(ref_audio_file), None)
|
||||
@@ -7,11 +7,9 @@ Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2t
|
||||
- empty ``source_outputs`` raises
|
||||
- latent fallback to ``hidden_states`` when ``multimodal_output`` is empty
|
||||
- both inputs missing -> raises
|
||||
- additional_information payload carries the keys the talker expects
|
||||
(prompt_embeds, prompt_token_ids, llm_output_token_ids, llm_output_text)
|
||||
- dummy talker prompt ``[BOS, PAD, EOS] = [1, 0, 2]`` (single prefill step)
|
||||
- structured model_intermediate_buffer carries the thinker ids and text
|
||||
- scheduler prompt tokens follow the selected TTS region or output tokens
|
||||
- MiniCPM-o 4.5 TTS region detection on 151703 / 151704 tokens
|
||||
- MiniCPM-o 2.6 fallback detection on 151691 / 151692 when no 4.5 markers
|
||||
- No TTS markers present -> no ``tts_token_ids`` / ``tts_hidden_states`` keys
|
||||
- prompt arg is normalized to a list and ``multi_modal_data`` is gated by
|
||||
``requires_multimodal_data``
|
||||
@@ -24,6 +22,9 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -32,6 +33,55 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
_HIDDEN_DIM = 4
|
||||
|
||||
|
||||
def test_native_duplex_talker_emits_empty_text_metadata_to_clear_previous_segment() -> None:
|
||||
class _Talker:
|
||||
_ar_last_chunk_flags = [True]
|
||||
_ar_turn_end_flags = [True]
|
||||
|
||||
def __call__(self, **kwargs):
|
||||
del kwargs
|
||||
return None, torch.zeros(0, dtype=torch.float32)
|
||||
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.model_stage = "tts"
|
||||
model.config = SimpleNamespace(hidden_size=_HIDDEN_DIM)
|
||||
model.talker = _Talker()
|
||||
|
||||
output = model.forward(
|
||||
input_ids=torch.tensor([1]),
|
||||
positions=torch.tensor([0]),
|
||||
runtime_additional_information=[
|
||||
{
|
||||
"native_duplex": True,
|
||||
"duplex": {"turn_id": 1, "epoch": 0},
|
||||
"meta": {"native_duplex_segment_text": ""},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert output.multimodal_outputs is not None
|
||||
assert output.multimodal_outputs["meta.llm_output_text_utf8"].numel() == 0
|
||||
assert output.multimodal_outputs["meta.audio_text_total_chars"].tolist() == [0]
|
||||
|
||||
|
||||
def test_tts_embed_input_ids_uses_active_talker_embedding(monkeypatch) -> None:
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.model_stage = "tts"
|
||||
expected = torch.ones((2, _HIDDEN_DIM))
|
||||
|
||||
monkeypatch.setattr(
|
||||
model,
|
||||
"get_input_embeddings",
|
||||
lambda input_ids: expected,
|
||||
)
|
||||
|
||||
actual = model.embed_input_ids(torch.tensor([11, 12]))
|
||||
|
||||
assert actual is expected
|
||||
|
||||
|
||||
def _make_thinker_output(
|
||||
*,
|
||||
prompt_token_ids: list[int],
|
||||
@@ -40,6 +90,7 @@ def _make_thinker_output(
|
||||
request_id: str = "req-0",
|
||||
latent: torch.Tensor | None = None,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
multimodal_output: dict[str, object] | None = None,
|
||||
):
|
||||
"""Construct a minimal mock of a thinker engine output entry.
|
||||
|
||||
@@ -48,8 +99,11 @@ def _make_thinker_output(
|
||||
- per-output: ``multimodal_output`` (dict), ``hidden_states`` (opt),
|
||||
``text``, ``token_ids``
|
||||
"""
|
||||
mm_output = dict(multimodal_output or {})
|
||||
if latent is not None:
|
||||
mm_output["latent"] = latent
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"latent": latent} if latent is not None else {},
|
||||
multimodal_output=mm_output,
|
||||
token_ids=output_token_ids,
|
||||
text=text,
|
||||
)
|
||||
@@ -90,17 +144,16 @@ class TestBasicShape:
|
||||
)
|
||||
assert len(out) == 2
|
||||
|
||||
def test_talker_prompt_token_ids_dummy_bos_pad_eos(self) -> None:
|
||||
def test_talker_scheduler_prompt_uses_output_tokens(self) -> None:
|
||||
hidden = torch.zeros((2, _HIDDEN_DIM))
|
||||
out = llm2tts(
|
||||
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
|
||||
prompt=None,
|
||||
)
|
||||
# The talker AR framework needs *some* tokens to do a single prefill;
|
||||
# this is the agreed [BOS, PAD, EOS] minimal payload.
|
||||
assert out[0]["prompt_token_ids"] == [1, 0, 2]
|
||||
assert out[0]["prompt_token_ids"] == [20]
|
||||
assert "stream_output" not in out[0]["model_intermediate_buffer"]
|
||||
|
||||
def test_additional_information_carries_thinker_outputs(self) -> None:
|
||||
def test_model_intermediate_buffer_carries_thinker_outputs(self) -> None:
|
||||
prompt_ids = [10, 11, 12]
|
||||
out_ids = [20, 21]
|
||||
hidden = torch.randn(len(prompt_ids) + len(out_ids), _HIDDEN_DIM)
|
||||
@@ -116,21 +169,17 @@ class TestBasicShape:
|
||||
],
|
||||
prompt=None,
|
||||
)
|
||||
ai = result[0]["additional_information"]
|
||||
assert ai["prompt_token_ids"] == prompt_ids
|
||||
assert ai["llm_output_token_ids"] == out_ids
|
||||
assert ai["llm_output_text"] == ["hello"]
|
||||
# prompt_embeds = float32 view of the prompt-portion of hidden states.
|
||||
assert ai["prompt_embeds"].dtype == torch.float32
|
||||
assert ai["prompt_embeds"].shape == (len(prompt_ids), _HIDDEN_DIM)
|
||||
assert torch.equal(ai["prompt_embeds"], hidden[: len(prompt_ids)].to(torch.float32))
|
||||
buffer = result[0]["model_intermediate_buffer"]
|
||||
assert buffer["ids"]["prompt"] == prompt_ids
|
||||
assert buffer["ids"]["output"] == out_ids
|
||||
assert buffer["llm_output_text"] == ["hello"]
|
||||
|
||||
def test_latent_in_multimodal_output_takes_precedence(self) -> None:
|
||||
# When both ``multimodal_output["latent"]`` and ``hidden_states`` are
|
||||
# present, the latent payload must win (this is the steady-state path
|
||||
# produced by the thinker stage).
|
||||
prompt_ids = [10, 11]
|
||||
out_ids = [20]
|
||||
out_ids = [151703, 30, 151704]
|
||||
latent = torch.ones((len(prompt_ids) + len(out_ids), _HIDDEN_DIM))
|
||||
hidden = torch.zeros_like(latent)
|
||||
|
||||
@@ -145,19 +194,17 @@ class TestBasicShape:
|
||||
],
|
||||
prompt=None,
|
||||
)
|
||||
ai = result[0]["additional_information"]
|
||||
buffer = result[0]["model_intermediate_buffer"]
|
||||
# latent (ones) won over hidden_states (zeros)
|
||||
assert torch.equal(ai["prompt_embeds"], latent[: len(prompt_ids)].to(torch.float32))
|
||||
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), latent[3:4].to(torch.float32))
|
||||
|
||||
|
||||
class TestTtsRegionDetection:
|
||||
"""The bridge auto-detects which MiniCPM-o generation produced the output
|
||||
by sniffing for 4.5-specific BOS/EOS token ids (151703 / 151704); when
|
||||
those are absent it falls back to the 2.6 ids (151691 / 151692).
|
||||
"""The bridge detects MiniCPM-o 4.5 BOS/EOS ids (151703 / 151704).
|
||||
|
||||
This is a regression guard: a single off-by-one or wrong branch
|
||||
precedence here causes the talker to receive an empty / wrong-region
|
||||
slice and emit silent audio.
|
||||
causes the talker to receive an empty / wrong-region slice and emit
|
||||
silent audio.
|
||||
"""
|
||||
|
||||
def _run(self, prompt_token_ids, output_token_ids):
|
||||
@@ -173,7 +220,7 @@ class TestTtsRegionDetection:
|
||||
],
|
||||
prompt=None,
|
||||
)
|
||||
return result[0]["additional_information"], hidden
|
||||
return result[0]["model_intermediate_buffer"], hidden
|
||||
|
||||
def test_4_5_markers_detected(self) -> None:
|
||||
# prompt: [10, 11]
|
||||
@@ -181,45 +228,26 @@ class TestTtsRegionDetection:
|
||||
# full sequence: [10, 11, 151703, 30, 31, 151704, 40]
|
||||
# 0 1 2 3 4 5 6
|
||||
# 4.5 BOS at idx 2 -> slice starts at 3; EOS at idx 5 -> slice ends at 5.
|
||||
ai, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40])
|
||||
assert "tts_token_ids" in ai and "tts_hidden_states" in ai
|
||||
assert ai["tts_token_ids"].tolist() == [30, 31]
|
||||
assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
|
||||
|
||||
def test_4_5_takes_precedence_when_both_markers_present(self) -> None:
|
||||
# If both 2.6 and 4.5 markers appear (shouldn't really happen on a
|
||||
# real checkpoint, but exercises the loop's break), the 4.5 markers
|
||||
# win because the source loop pins on the first 4.5 id seen.
|
||||
# sequence index: 0 1 2 3 4 5 6 7 8
|
||||
ai, hidden = self._run([10], [151691, 70, 151692, 151703, 30, 151704, 40])
|
||||
assert ai["tts_token_ids"].tolist() == [30]
|
||||
assert torch.equal(ai["tts_hidden_states"], hidden[5:6])
|
||||
|
||||
def test_2_6_fallback(self) -> None:
|
||||
# No 4.5 markers anywhere -> use 2.6 BOS/EOS pair.
|
||||
# full sequence: [10, 151691, 30, 31, 151692, 40]
|
||||
# 0 1 2 3 4 5
|
||||
# BOS at idx 1 -> slice starts at 2; EOS at idx 4 -> slice ends at 4.
|
||||
ai, hidden = self._run([10], [151691, 30, 31, 151692, 40])
|
||||
assert ai["tts_token_ids"].tolist() == [30, 31]
|
||||
assert torch.equal(ai["tts_hidden_states"], hidden[2:4])
|
||||
buffer, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40])
|
||||
assert buffer["ids"]["tts"] == [30, 31]
|
||||
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5])
|
||||
|
||||
def test_bos_without_eos_runs_to_end(self) -> None:
|
||||
# When BOS is found but EOS is missing (typical for an in-flight or
|
||||
# truncated decode), the slice should extend to the end of the
|
||||
# hidden-state matrix instead of being dropped.
|
||||
# sequence: [10, 11, 151703, 30, 31]
|
||||
ai, hidden = self._run([10, 11], [151703, 30, 31])
|
||||
assert ai["tts_token_ids"].tolist() == [30, 31]
|
||||
assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
|
||||
buffer, hidden = self._run([10, 11], [151703, 30, 31])
|
||||
assert buffer["ids"]["tts"] == [30, 31]
|
||||
assert torch.equal(torch.tensor(buffer["hidden_states"]["tts"]), hidden[3:5])
|
||||
|
||||
def test_no_tts_markers_omits_slice_keys(self) -> None:
|
||||
# If neither marker pair is present, the bridge should NOT populate
|
||||
# ``tts_token_ids`` / ``tts_hidden_states`` — the talker should fall
|
||||
# through to the dummy path.
|
||||
ai, _ = self._run([10, 11], [20, 21, 22])
|
||||
assert "tts_token_ids" not in ai
|
||||
assert "tts_hidden_states" not in ai
|
||||
buffer, _ = self._run([10, 11], [20, 21, 22])
|
||||
assert "tts" not in buffer["ids"]
|
||||
assert "tts" not in buffer.get("hidden_states", {})
|
||||
|
||||
|
||||
class TestPromptAndMultiModal:
|
||||
@@ -251,14 +279,11 @@ class TestPromptAndMultiModal:
|
||||
)
|
||||
assert out[0]["multi_modal_data"] == mm
|
||||
|
||||
def test_streaming_context_is_accepted_and_ignored(self) -> None:
|
||||
# The framework passes ``streaming_context`` for streaming-capable
|
||||
# pipelines; MiniCPM-o 4.5 is turn-taking so the bridge must accept
|
||||
# but discard it without crashing.
|
||||
def test_internal_streaming_context_is_accepted(self) -> None:
|
||||
hidden = torch.zeros((2, _HIDDEN_DIM))
|
||||
out = llm2tts(
|
||||
[_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
|
||||
prompt=None,
|
||||
streaming_context=object(),
|
||||
_streaming_context=object(),
|
||||
)
|
||||
assert len(out) == 1
|
||||
@@ -44,6 +44,20 @@ class TestRegistryDeclaration:
|
||||
assert pipeline.model_type == _PIPELINE_KEY
|
||||
assert pipeline.model_arch == "MiniCPMO45OmniForConditionalGeneration"
|
||||
|
||||
def test_native_duplex_control_is_explicit_without_a_fixed_session_cap(self) -> None:
|
||||
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
|
||||
assert pipeline.duplex_control_enabled is True
|
||||
assert pipeline.duplex_serving_adapter == (
|
||||
"vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
|
||||
)
|
||||
assert not hasattr(pipeline, "max_native_duplex_sessions")
|
||||
|
||||
def test_ordinary_pipeline_defaults_to_no_duplex_control(self) -> None:
|
||||
pipeline = PipelineConfig(model_type="ordinary")
|
||||
assert pipeline.duplex_control_enabled is False
|
||||
assert pipeline.duplex_serving_adapter is None
|
||||
assert not hasattr(pipeline, "max_native_duplex_sessions")
|
||||
|
||||
|
||||
class TestPipelineTopology:
|
||||
@pytest.fixture(scope="class")
|
||||
|
||||
Reference in new issue
Block a user