[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-05 08:20:21 +00:00
1 parent d4096e01fe
commit 5d7fdf9350
179 files changed
+6462 -3832

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import pytest
from vllm_omni.entrypoints.duplex.commit_policy import (
CommitAction,
CommitSnapshot,
decide_commit_action,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
("case", "snapshot", "expected"),
[
(
"first idle commit",
CommitSnapshot(
auto_responds=True,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=False,
playback_active=False,
),
CommitAction.START_AUTO_RESPONSE,
),
(
"native auto-response remains model-owned during playback",
CommitSnapshot(
auto_responds=True,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=True,
playback_active=True,
),
CommitAction.START_AUTO_RESPONSE,
),
(
"native auto-response commit without speech",
CommitSnapshot(
auto_responds=True,
speech_since_commit=False,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=True,
playback_active=False,
),
CommitAction.COMMIT_ONLY,
),
(
"non-auto response overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=True,
playback_active=False,
),
CommitAction.DEFER_ACTIVE_RESPONSE,
),
(
"non-auto playback overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=800,
native_response_in_progress=False,
playback_active=True,
),
CommitAction.DEFER_ACTIVE_RESPONSE,
),
(
"non-auto commit without overlap",
CommitSnapshot(
auto_responds=False,
speech_since_commit=True,
active_response_id=None,
overlap_speech_ms=0,
native_response_in_progress=False,
playback_active=False,
),
CommitAction.COMMIT_ONLY,
),
],
ids=lambda value: value if isinstance(value, str) else None,
)
def test_decide_commit_action_covers_realtime_lifecycle_sequences(
case: str,
snapshot: CommitSnapshot,
expected: CommitAction,
) -> None:
del case
assert decide_commit_action(snapshot) is expected
@@ -0,0 +1,179 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
import numpy as np
import pytest
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import MiniCPMO45PcmAppendBuffer
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def pcm_payload(samples: int, *, speech: bool = True) -> dict[str, object]:
audio = np.ones(samples, dtype=np.float32).tobytes()
return {
"type": "audio",
"audio": base64.b64encode(audio).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": 16_000,
"is_speech": speech,
}
def test_commit_does_not_add_silence_after_incremental_audio_was_drained():
buffer = MiniCPMO45PcmAppendBuffer()
emitted = buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
committed = buffer.commit(chunk_period_ms=1_000)
assert emitted is not None
assert not buffer.has_pending()
assert committed is None
def test_append_emits_one_model_unit_when_multiple_units_are_buffered():
buffer = MiniCPMO45PcmAppendBuffer()
emitted = buffer.append(pcm_payload(32_000), chunk_period_ms=1_000)
assert emitted is not None
assert len(base64.b64decode(emitted["audio"])) == 16_000 * 4
assert buffer.pending_byte_count == 16_000 * 4
def test_speech_marker_does_not_leak_across_irregular_chunk_boundaries():
buffer = MiniCPMO45PcmAppendBuffer()
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
speech_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
silence_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
assert speech_unit is not None
assert speech_unit["is_speech"] is True
assert silence_unit is not None
assert silence_unit["is_speech"] is False
def test_force_listen_marker_does_not_leak_across_irregular_chunk_boundaries():
buffer = MiniCPMO45PcmAppendBuffer()
forced = pcm_payload(15_000, speech=False)
forced["force_listen"] = True
assert buffer.append(forced, chunk_period_ms=1_000) is None
forced_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
unforced_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
assert forced_unit is not None
assert forced_unit["force_listen"] is True
assert unforced_unit is not None
assert unforced_unit["force_listen"] is False
def test_pcm_append_rollback_restores_per_span_speech_markers():
buffer = MiniCPMO45PcmAppendBuffer()
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
reservation = buffer.prepare_append(
pcm_payload(2_000, speech=False),
operation_id="mixed-unit",
chunk_period_ms=1_000,
)
assert reservation is not None
reservation.rollback()
first = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
second = buffer.flush(chunk_period_ms=1_000)
assert first is not None
assert first["is_speech"] is True
assert second is not None
assert second["is_speech"] is False
def test_commit_without_speech_does_not_synthesize_terminal_audio():
buffer = MiniCPMO45PcmAppendBuffer()
buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000)
committed = buffer.commit(chunk_period_ms=1_000)
assert committed is None
def test_commit_resets_cumulative_turn_accounting():
buffer = MiniCPMO45PcmAppendBuffer()
buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
buffer.commit(chunk_period_ms=1_000)
empty = buffer.commit(chunk_period_ms=1_000)
assert empty is None
def test_pcm_append_reservation_rollback_restores_emitted_audio():
buffer = MiniCPMO45PcmAppendBuffer()
original = pcm_payload(16_000)
reservation = buffer.prepare_append(
original,
operation_id="append-1",
chunk_period_ms=1_000,
)
assert reservation is not None
assert not buffer.has_pending()
reservation.rollback()
assert buffer.has_pending()
retried = buffer.flush(chunk_period_ms=1_000)
assert retried is not None
assert base64.b64decode(retried["audio"]) == base64.b64decode(original["audio"])
def test_pcm_commit_keeps_prior_append_reservation_active():
buffer = MiniCPMO45PcmAppendBuffer()
append_reservation = buffer.prepare_append(
pcm_payload(16_000),
operation_id="append-before-commit",
chunk_period_ms=1_000,
)
commit_reservation = buffer.prepare_commit(
operation_id="commit-after-append",
chunk_period_ms=1_000,
)
assert append_reservation is not None
assert append_reservation.active
assert commit_reservation.payload is None
append_reservation.commit()
commit_reservation.commit()
def test_pcm_commit_reservation_rollback_restores_residual_audio():
buffer = MiniCPMO45PcmAppendBuffer()
original = pcm_payload(8_000)
assert (
buffer.prepare_append(
original,
operation_id="buffer-half-chunk",
chunk_period_ms=1_000,
)
is None
)
reservation = buffer.prepare_commit(
operation_id="final-half-chunk",
chunk_period_ms=1_000,
)
assert reservation.payload is not None
assert reservation.payload["final"] is True
reservation.rollback()
retried = buffer.prepare_commit(
operation_id="retry-final-half-chunk",
chunk_period_ms=1_000,
)
assert retried.payload is not None
assert base64.b64decode(retried.payload["audio"]) == (base64.b64decode(original["audio"]) + b"\x00" * (8_000 * 4))
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
"Speak exactly.",
True,
"The quick brown fox.",
)
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
assert suffix == (
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
)